From a61b1c4b0d17a4058da63f3b605d8f900d42d210 Mon Sep 17 00:00:00 2001 From: Willow Ahrens Date: Thu, 27 Aug 2026 13:16:56 -0400 Subject: [PATCH 01/16] start with reformatter --- compliance/reformat.c | 620 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 620 insertions(+) create mode 100644 compliance/reformat.c diff --git a/compliance/reformat.c b/compliance/reformat.c new file mode 100644 index 0000000..3b8862c --- /dev/null +++ b/compliance/reformat.c @@ -0,0 +1,620 @@ +/* + * SPDX-FileCopyrightText: 2026 Binsparse Developers + * SPDX-License-Identifier: BSD-3-Clause + * + * Reformat a Binsparse matrix or tensor through a coordinate list. + * + * Usage: reformat INPUT OUTPUT TARGET_HEADER.json [COMPRESSION] + * + * INPUT may use a predefined matrix/vector alias or the custom tensor format. + * TARGET_HEADER is a Binsparse header (the object containing "format", not the + * outer {"binsparse": ...} wrapper). Both predefined aliases and arbitrary + * custom dense/sparse level trees are accepted. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +bsp_error_t bsp_read_attribute(char **attribute, hid_t f, char *name); + +typedef struct { + size_t *coord; + size_t value; +} entry_t; + +typedef struct { + entry_t *entry; + size_t size; + size_t capacity; + int rank; +} entries_t; + +static void die(const char *message) { + fprintf(stderr, "reformat: %s\n", message); + exit(EXIT_FAILURE); +} + +static cJSON *read_json(const char *path) { + FILE *file = fopen(path, "rb"); + long size = 0; + char *text; + cJSON *json; + if (file == NULL) die(strerror(errno)); + if (fseek(file, 0, SEEK_END) != 0 || (size = ftell(file)) < 0 || + fseek(file, 0, SEEK_SET) != 0) + die("cannot read target header"); + text = malloc((size_t) size + 1); + if (text == NULL) die("out of memory"); + if (fread(text, 1, (size_t) size, file) != (size_t) size) + die("cannot read target header"); + fclose(file); + text[size] = '\0'; + json = cJSON_Parse(text); + free(text); + if (!cJSON_IsObject(json)) die("target header must be a JSON object"); + return json; +} + +static cJSON *input_header(const char *path) { + hid_t file = H5Fopen(path, H5F_ACC_RDONLY, H5P_DEFAULT); + char *text = NULL; + cJSON *outer, *header; + if (file < 0 || bsp_read_attribute(&text, file, "binsparse") != BSP_SUCCESS) + die("cannot read input Binsparse header"); + H5Fclose(file); + outer = cJSON_Parse(text); + free(text); + if (!cJSON_IsObject(outer)) die("invalid input Binsparse header"); + header = cJSON_DetachItemFromObjectCaseSensitive(outer, "binsparse"); + cJSON_Delete(outer); + if (!cJSON_IsObject(header)) die("input has no binsparse header"); + return header; +} + +static bool header_values_are_iso(cJSON *header) { + cJSON *types = cJSON_GetObjectItemCaseSensitive(header, "data_types"); + const char *value = cJSON_GetStringValue( + cJSON_GetObjectItemCaseSensitive(types, "values")); + return value != NULL && strncmp(value, "iso[", 4) == 0; +} + +static void push_entry(entries_t *entries, const size_t *coord, size_t value) { + entry_t *item; + if (entries->size == entries->capacity) { + size_t capacity = entries->capacity == 0 ? 16 : 2 * entries->capacity; + entry_t *grown = realloc(entries->entry, capacity * sizeof(*grown)); + if (grown == NULL) die("out of memory"); + entries->entry = grown; + entries->capacity = capacity; + } + item = &entries->entry[entries->size++]; + item->coord = malloc((size_t) entries->rank * sizeof(*item->coord)); + if (item->coord == NULL) die("out of memory"); + memcpy(item->coord, coord, (size_t) entries->rank * sizeof(*coord)); + item->value = value; +} + +static void free_entries(entries_t *entries) { + for (size_t i = 0; i < entries->size; ++i) free(entries->entry[i].coord); + free(entries->entry); +} + +/* Expand one custom level. begin/end identify the parent positions entering + * the level; coordinates are stored row-wise in coords. */ +static void level_to_coo(const bsp_level_t *level, const bsp_tensor_t *tensor, + size_t depth, size_t parents, const size_t *coords, + entries_t *out) { + if (level->kind == BSP_TENSOR_ELEMENT) { + const bsp_element_t *element = level->data; + if (parents != tensor->nnz) + die("element positions do not match number_of_stored_values"); + for (size_t p = 0; p < parents; ++p) + push_entry(out, coords + p * (size_t) tensor->rank, + tensor->is_iso ? 0 : p); + (void) element; + return; + } + + if (level->kind == BSP_TENSOR_DENSE) { + const bsp_dense_t *dense = level->data; + size_t width = 1, count, *next; + for (int d = 0; d < dense->rank; ++d) + width *= tensor->dims[depth + (size_t) d]; + count = parents * width; + next = calloc(count * (size_t) tensor->rank, sizeof(*next)); + if (next == NULL && count != 0) die("out of memory"); + for (size_t p = 0; p < parents; ++p) { + for (size_t suffix = 0; suffix < width; ++suffix) { + size_t q = p * width + suffix, n = suffix; + memcpy(next + q * (size_t) tensor->rank, + coords + p * (size_t) tensor->rank, + depth * sizeof(*next)); + for (int d = dense->rank - 1; d >= 0; --d) { + size_t dim = tensor->dims[depth + (size_t) d]; + next[q * (size_t) tensor->rank + depth + (size_t) d] = n % dim; + n /= dim; + } + } + } + level_to_coo(dense->child, tensor, depth + (size_t) dense->rank, count, + next, out); + free(next); + return; + } + + if (level->kind == BSP_TENSOR_SPARSE) { + const bsp_sparse_t *sparse = level->data; + size_t count = sparse->indices[0].size, *next; + size_t q = 0; + next = calloc(count * (size_t) tensor->rank, sizeof(*next)); + if (next == NULL && count != 0) die("out of memory"); + for (size_t p = 0; p < parents; ++p) { + size_t begin = 0, end = count; + if (sparse->pointers_to != NULL) { + bsp_array_t pointers = *sparse->pointers_to; + bsp_array_read(pointers, p, begin); + bsp_array_read(pointers, p + 1, end); + } + for (size_t i = begin; i < end; ++i, ++q) { + memcpy(next + q * (size_t) tensor->rank, + coords + p * (size_t) tensor->rank, + depth * sizeof(*next)); + for (int d = 0; d < sparse->rank; ++d) { + size_t index = 0; + bsp_array_read(sparse->indices[d], i, index); + if (index >= tensor->dims[depth + (size_t) d]) + die("input coordinate is out of bounds"); + next[q * (size_t) tensor->rank + depth + (size_t) d] = index; + } + } + } + if (q != count) die("invalid sparse pointers"); + level_to_coo(sparse->child, tensor, depth + (size_t) sparse->rank, count, + next, out); + free(next); + return; + } + die("unknown tensor level"); +} + +static entries_t tensor_to_coo(const bsp_tensor_t *tensor) { + entries_t result = {0}; + size_t *root; + result.rank = tensor->rank; + root = calloc((size_t) tensor->rank, sizeof(*root)); + if (root == NULL) die("out of memory"); + level_to_coo(tensor->level, tensor, 0, 1, root, &result); + free(root); + return result; +} + +static entries_t matrix_to_coo(bsp_matrix_t *matrix) { + entries_t result = {0}; + result.rank = bsp_matrix_format_is_vector(matrix->format) ? 1 : 2; + if (matrix->format == BSP_COOR || matrix->format == BSP_COOC || + matrix->format == BSP_CVEC) { + for (size_t i = 0; i < matrix->nnz; ++i) { + size_t coord[2] = {0, 0}; + bsp_array_read(matrix->indices_0, i, coord[0]); + if (result.rank == 2) bsp_array_read(matrix->indices_1, i, coord[1]); + push_entry(&result, coord, matrix->is_iso ? 0 : i); + } + return result; + } + if (matrix->format == BSP_CSR || matrix->format == BSP_CSC) { + size_t fibers = matrix->format == BSP_CSR ? matrix->nrows : matrix->ncols; + for (size_t p = 0; p < fibers; ++p) { + size_t begin = 0, end = 0; + bsp_array_read(matrix->pointers_to_1, p, begin); + bsp_array_read(matrix->pointers_to_1, p + 1, end); + for (size_t i = begin; i < end; ++i) { + size_t coord[2] = {p, 0}; + bsp_array_read(matrix->indices_1, i, coord[1]); + push_entry(&result, coord, matrix->is_iso ? 0 : i); + } + } + return result; + } + if (matrix->format == BSP_DCSR || matrix->format == BSP_DCSC) { + for (size_t p = 0; p < matrix->indices_0.size; ++p) { + size_t begin = 0, end = 0, outer = 0; + bsp_array_read(matrix->indices_0, p, outer); + bsp_array_read(matrix->pointers_to_1, p, begin); + bsp_array_read(matrix->pointers_to_1, p + 1, end); + for (size_t i = begin; i < end; ++i) { + size_t coord[2] = {outer, 0}; + bsp_array_read(matrix->indices_1, i, coord[1]); + push_entry(&result, coord, matrix->is_iso ? 0 : i); + } + } + return result; + } + if (matrix->format == BSP_DVEC || matrix->format == BSP_DMAT || + matrix->format == BSP_DMATC) { + size_t rows = matrix->format == BSP_DMATC ? matrix->ncols : matrix->nrows; + size_t cols = result.rank == 1 ? 1 + : (matrix->format == BSP_DMATC + ? matrix->nrows + : matrix->ncols); + for (size_t i = 0; i < matrix->nnz; ++i) { + size_t coord[2] = {i / cols, i % cols}; + if (result.rank == 1) coord[0] = i; + if (coord[0] >= rows) die("dense value count exceeds shape"); + push_entry(&result, coord, matrix->is_iso ? 0 : i); + } + return result; + } + die("unsupported predefined input format"); + return result; +} + +/* A predefined format is only a named view of a custom level tree. */ +static bsp_error_t write_predefined(const char *path, bsp_tensor_t tensor, + const char *format, int compression) { + bsp_matrix_t matrix; + bsp_level_t *level = tensor.level; + bsp_construct_default_matrix_t(&matrix); + matrix.format = bsp_get_matrix_format((char *) format); + matrix.nrows = tensor.dims[0]; + matrix.ncols = tensor.rank == 1 ? 1 : tensor.dims[1]; + matrix.nnz = tensor.nnz; + matrix.is_iso = tensor.is_iso; + matrix.structure = tensor.structure; + matrix.values = bsp_get_tensor_values(tensor); + + if (matrix.format == BSP_INVALID_FORMAT) return BSP_ERROR_UNSUPPORTED; + if (matrix.format == BSP_DVEC || matrix.format == BSP_DMAT || + matrix.format == BSP_DMATC) { + return bsp_write_matrix(path, matrix, NULL, NULL, compression); + } + if (matrix.format == BSP_CVEC || matrix.format == BSP_COOR || + matrix.format == BSP_COOC) { + bsp_sparse_t *sparse; + if (level->kind != BSP_TENSOR_SPARSE) return BSP_ERROR_INTERNAL; + sparse = level->data; + matrix.indices_0 = sparse->indices[0]; + if (sparse->rank == 2) matrix.indices_1 = sparse->indices[1]; + return bsp_write_matrix(path, matrix, NULL, NULL, compression); + } + if (matrix.format == BSP_CSR || matrix.format == BSP_CSC) { + bsp_dense_t *dense; + bsp_sparse_t *sparse; + if (level->kind != BSP_TENSOR_DENSE) return BSP_ERROR_INTERNAL; + dense = level->data; + if (dense->child->kind != BSP_TENSOR_SPARSE) return BSP_ERROR_INTERNAL; + sparse = dense->child->data; + matrix.pointers_to_1 = *sparse->pointers_to; + matrix.indices_1 = sparse->indices[0]; + return bsp_write_matrix(path, matrix, NULL, NULL, compression); + } + if (matrix.format == BSP_DCSR || matrix.format == BSP_DCSC) { + bsp_sparse_t *outer, *inner; + if (level->kind != BSP_TENSOR_SPARSE) return BSP_ERROR_INTERNAL; + outer = level->data; + if (outer->child->kind != BSP_TENSOR_SPARSE) return BSP_ERROR_INTERNAL; + inner = outer->child->data; + matrix.indices_0 = outer->indices[0]; + matrix.pointers_to_1 = *inner->pointers_to; + matrix.indices_1 = inner->indices[0]; + return bsp_write_matrix(path, matrix, NULL, NULL, compression); + } + return BSP_ERROR_UNSUPPORTED; +} + +static int compare_rank; +static int compare_entry(const void *a_, const void *b_) { + const entry_t *a = a_, *b = b_; + for (int d = 0; d < compare_rank; ++d) { + if (a->coord[d] < b->coord[d]) return -1; + if (a->coord[d] > b->coord[d]) return 1; + } + return 0; +} + +static void apply_transpose(entries_t *entries, const size_t *source, + const size_t *target) { + size_t *copy = malloc((size_t) entries->rank * sizeof(*copy)); + if (copy == NULL) die("out of memory"); + for (size_t k = 0; k < entries->size; ++k) { + memcpy(copy, entries->entry[k].coord, + (size_t) entries->rank * sizeof(*copy)); + for (int td = 0; td < entries->rank; ++td) { + int sd = 0; + while (sd < entries->rank && source[sd] != target[td]) ++sd; + if (sd == entries->rank) die("transpose is not a permutation"); + entries->entry[k].coord[td] = copy[sd]; + } + } + free(copy); + compare_rank = entries->rank; + qsort(entries->entry, entries->size, sizeof(*entries->entry), compare_entry); +} + +static bsp_array_t copy_values(const bsp_array_t source, const entries_t *entries, + bool iso) { + bsp_array_t values; + size_t size = iso && entries->size != 0 ? 1 : entries->size; + if (bsp_construct_array_t(&values, size, source.type) != BSP_SUCCESS) + die("out of memory"); + for (size_t i = 0; i < size; ++i) + memcpy((char *) values.data + i * bsp_type_size(values.type), + (char *) source.data + entries->entry[iso ? 0 : i].value * + bsp_type_size(source.type), + bsp_type_size(source.type)); + return values; +} + +static size_t prefix_equal(const entry_t *a, const entry_t *b, size_t depth, + int rank) { + for (int d = 0; d < rank; ++d) + if (a->coord[depth + (size_t) d] != b->coord[depth + (size_t) d]) return 0; + return 1; +} + +/* Recursively build a level. parent_ptr partitions entries into incoming + * fibers. The entries are already lexicographically sorted. */ +static bsp_level_t *build_level(cJSON *description, const entries_t *entries, + const size_t *dims, size_t depth, + const size_t *parent_ptr, size_t parents, + bool root, bsp_array_t values) { + cJSON *desc = cJSON_GetObjectItemCaseSensitive(description, "level_desc"); + bsp_level_t *level = calloc(1, sizeof(*level)); + const char *name = cJSON_GetStringValue(desc); + if (level == NULL) die("out of memory"); + if (name != NULL && strcmp(name, "element") == 0) { + bsp_element_t *element = malloc(sizeof(*element)); + for (size_t p = 0; p < parents; ++p) + if (parent_ptr[p + 1] - parent_ptr[p] != 1) + die("each element position must have one value"); + if (element == NULL) die("out of memory"); + element->values = values; + level->kind = BSP_TENSOR_ELEMENT; + level->data = element; + return level; + } + cJSON *rank_json = cJSON_GetObjectItemCaseSensitive(description, "rank"); + int rank = (int) cJSON_GetNumberValue(rank_json); + cJSON *child_desc = cJSON_GetObjectItemCaseSensitive(description, "level"); + if (rank <= 0 || depth + (size_t) rank > (size_t) entries->rank || + !cJSON_IsObject(child_desc)) + die("invalid target level descriptor"); + + if (strcmp(name ? name : "", "sparse") == 0) { + bsp_sparse_t *sparse = calloc(1, sizeof(*sparse)); + size_t groups = 0, *child_ptr; + if (sparse == NULL) die("out of memory"); + for (size_t p = 0; p < parents; ++p) + for (size_t i = parent_ptr[p]; i < parent_ptr[p + 1]; ++i) + if (i == parent_ptr[p] || + !prefix_equal(&entries->entry[i - 1], &entries->entry[i], depth, + rank)) + ++groups; + sparse->rank = rank; + sparse->indices = calloc((size_t) rank, sizeof(*sparse->indices)); + child_ptr = malloc((groups + 1) * sizeof(*child_ptr)); + if (sparse->indices == NULL || child_ptr == NULL) die("out of memory"); + for (int d = 0; d < rank; ++d) + if (bsp_construct_array_t(&sparse->indices[d], groups, + bsp_pick_integer_type(dims[depth + (size_t) d])) != + BSP_SUCCESS) + die("out of memory"); + if (!root) { + sparse->pointers_to = malloc(sizeof(*sparse->pointers_to)); + if (sparse->pointers_to == NULL || + bsp_construct_array_t(sparse->pointers_to, parents + 1, + bsp_pick_integer_type(groups)) != BSP_SUCCESS) + die("out of memory"); + } + groups = 0; + child_ptr[0] = 0; + for (size_t p = 0; p < parents; ++p) { + if (!root) { + bsp_array_t pointers = *sparse->pointers_to; + bsp_array_write(pointers, p, groups); + } + for (size_t i = parent_ptr[p]; i < parent_ptr[p + 1]; ++i) { + if (i == parent_ptr[p] || + !prefix_equal(&entries->entry[i - 1], &entries->entry[i], depth, + rank)) { + if (groups != 0) child_ptr[groups] = i; + for (int d = 0; d < rank; ++d) + bsp_array_write(sparse->indices[d], groups, + entries->entry[i].coord[depth + (size_t) d]); + ++groups; + } + } + } + child_ptr[groups] = entries->size; + if (!root) { + bsp_array_t pointers = *sparse->pointers_to; + bsp_array_write(pointers, parents, groups); + } + sparse->child = build_level(child_desc, entries, dims, depth + (size_t) rank, + child_ptr, groups, false, values); + free(child_ptr); + level->kind = BSP_TENSOR_SPARSE; + level->data = sparse; + return level; + } + + if (strcmp(name ? name : "", "dense") == 0) { + bsp_dense_t *dense = malloc(sizeof(*dense)); + size_t width = 1, children, *child_ptr; + if (dense == NULL) die("out of memory"); + for (int d = 0; d < rank; ++d) width *= dims[depth + (size_t) d]; + children = parents * width; + child_ptr = malloc((children + 1) * sizeof(*child_ptr)); + if (child_ptr == NULL) die("out of memory"); + size_t at = 0; + for (size_t p = 0; p < parents; ++p) { + size_t end = parent_ptr[p + 1]; + for (size_t suffix = 0; suffix < width; ++suffix) { + child_ptr[p * width + suffix] = at; + while (at < end) { + size_t n = suffix; + int equal = 1; + for (int d = rank - 1; d >= 0; --d) { + size_t dim = dims[depth + (size_t) d]; + if (entries->entry[at].coord[depth + (size_t) d] != n % dim) + equal = 0; + n /= dim; + } + if (!equal) break; + ++at; + } + } + } + child_ptr[children] = entries->size; + dense->rank = rank; + dense->child = build_level(child_desc, entries, dims, + depth + (size_t) rank, child_ptr, children, + false, values); + free(child_ptr); + level->kind = BSP_TENSOR_DENSE; + level->data = dense; + return level; + } + die("unknown target level descriptor"); + return NULL; +} + +static cJSON *predefined_level(const char *format, size_t *transpose, int rank) { + const char *json = NULL; + if (!strcmp(format, "DVEC")) json = "{\"level_desc\":\"dense\",\"rank\":1,\"level\":{\"level_desc\":\"element\"}}"; + else if (!strcmp(format, "CVEC")) json = "{\"level_desc\":\"sparse\",\"rank\":1,\"level\":{\"level_desc\":\"element\"}}"; + else if (!strcmp(format, "DMAT") || !strcmp(format, "DMATR")) json = "{\"level_desc\":\"dense\",\"rank\":2,\"level\":{\"level_desc\":\"element\"}}"; + else if (!strcmp(format, "DMATC")) { json = "{\"level_desc\":\"dense\",\"rank\":2,\"level\":{\"level_desc\":\"element\"}}"; transpose[0]=1; transpose[1]=0; } + else if (!strcmp(format, "COO") || !strcmp(format, "COOR")) json = "{\"level_desc\":\"sparse\",\"rank\":2,\"level\":{\"level_desc\":\"element\"}}"; + else if (!strcmp(format, "COOC")) { json = "{\"level_desc\":\"sparse\",\"rank\":2,\"level\":{\"level_desc\":\"element\"}}"; transpose[0]=1; transpose[1]=0; } + else if (!strcmp(format, "CSR")) json = "{\"level_desc\":\"dense\",\"rank\":1,\"level\":{\"level_desc\":\"sparse\",\"rank\":1,\"level\":{\"level_desc\":\"element\"}}}"; + else if (!strcmp(format, "CSC")) { json = "{\"level_desc\":\"dense\",\"rank\":1,\"level\":{\"level_desc\":\"sparse\",\"rank\":1,\"level\":{\"level_desc\":\"element\"}}}"; transpose[0]=1; transpose[1]=0; } + else if (!strcmp(format, "DCSR")) json = "{\"level_desc\":\"sparse\",\"rank\":1,\"level\":{\"level_desc\":\"sparse\",\"rank\":1,\"level\":{\"level_desc\":\"element\"}}}"; + else if (!strcmp(format, "DCSC")) { json = "{\"level_desc\":\"sparse\",\"rank\":1,\"level\":{\"level_desc\":\"sparse\",\"rank\":1,\"level\":{\"level_desc\":\"element\"}}}"; transpose[0]=1; transpose[1]=0; } + if (json == NULL || (rank != 1 && rank != 2)) return NULL; + return cJSON_Parse(json); +} + +static void usage(void) { + fprintf(stderr, "usage: reformat INPUT OUTPUT TARGET_HEADER.json [COMPRESSION]\n"); + exit(2); +} + +int main(int argc, char **argv) { + cJSON *source_header, *target_header, *format_json, *custom, *description; + const char *source_format, *target_format; + bsp_tensor_t tensor = bsp_construct_default_tensor_t(); + bsp_matrix_t matrix; + bsp_array_t source_values, values; + entries_t entries; + size_t *source_transpose, *target_transpose, *stored_dims, root_ptr[2]; + int rank, compression = 0; + bool source_is_tensor, iso; + if (argc < 4 || argc > 5) usage(); + if (argc == 5) compression = atoi(argv[4]); + if (compression < 0 || compression > 9) die("compression must be 0 through 9"); + + source_header = input_header(argv[1]); + target_header = read_json(argv[3]); + format_json = cJSON_GetObjectItemCaseSensitive(source_header, "format"); + source_format = cJSON_GetStringValue(format_json); + target_format = cJSON_GetStringValue(cJSON_GetObjectItemCaseSensitive(target_header, "format")); + if (source_format == NULL || target_format == NULL) die("header has no format string"); + source_is_tensor = strcmp(source_format, "custom") == 0; + + if (source_is_tensor) { + tensor = bsp_read_tensor(argv[1], NULL); + if (tensor.level == NULL) die("failed to parse input tensor"); + tensor.is_iso = header_values_are_iso(source_header); + rank = tensor.rank; + source_values = bsp_get_tensor_values(tensor); + iso = tensor.is_iso; + entries = tensor_to_coo(&tensor); + } else { + if (bsp_read_matrix(&matrix, argv[1], NULL) != BSP_SUCCESS) + die("failed to parse input matrix"); + rank = bsp_matrix_format_is_vector(matrix.format) ? 1 : 2; + source_values = matrix.values; + iso = matrix.is_iso; + entries = matrix_to_coo(&matrix); + } + + source_transpose = malloc((size_t) rank * sizeof(*source_transpose)); + target_transpose = malloc((size_t) rank * sizeof(*target_transpose)); + stored_dims = malloc((size_t) rank * sizeof(*stored_dims)); + if (!source_transpose || !target_transpose || !stored_dims) die("out of memory"); + for (int d = 0; d < rank; ++d) { + source_transpose[d] = source_is_tensor && tensor.transpose ? tensor.transpose[d] : (size_t) d; + target_transpose[d] = (size_t) d; + } + if (!source_is_tensor && rank == 2 && + (matrix.format == BSP_DMATC || matrix.format == BSP_CSC || + matrix.format == BSP_DCSC || matrix.format == BSP_COOC)) { + source_transpose[0] = 1; + source_transpose[1] = 0; + } + + if (!strcmp(target_format, "custom")) { + custom = cJSON_GetObjectItemCaseSensitive(target_header, "custom"); + description = cJSON_GetObjectItemCaseSensitive(custom, "level"); + cJSON *transpose = cJSON_GetObjectItemCaseSensitive(custom, "transpose"); + if (!cJSON_IsObject(description)) die("custom target requires custom.level"); + if (transpose != NULL) { + if (cJSON_GetArraySize(transpose) != rank) die("transpose rank mismatch"); + for (int d = 0; d < rank; ++d) + target_transpose[d] = (size_t) cJSON_GetNumberValue(cJSON_GetArrayItem(transpose, d)); + } + description = cJSON_Duplicate(description, 1); + } else { + description = predefined_level(target_format, target_transpose, rank); + if (description == NULL) die("unknown or rank-incompatible target format"); + } + + apply_transpose(&entries, source_transpose, target_transpose); + for (int d = 0; d < rank; ++d) { + size_t logical = target_transpose[d]; + stored_dims[d] = source_is_tensor ? tensor.dims[logical] + : (logical == 0 ? matrix.nrows : matrix.ncols); + } + values = copy_values(source_values, &entries, iso); + bsp_tensor_t result = bsp_construct_default_tensor_t(); + result.rank = rank; + result.nnz = entries.size; + result.is_iso = iso; + result.structure = source_is_tensor ? tensor.structure : matrix.structure; + result.dims = malloc((size_t) rank * sizeof(*result.dims)); + result.transpose = malloc((size_t) rank * sizeof(*result.transpose)); + if (!result.dims || !result.transpose) die("out of memory"); + for (int d = 0; d < rank; ++d) { + result.dims[d] = source_is_tensor ? tensor.dims[d] : (d == 0 ? matrix.nrows : matrix.ncols); + result.transpose[d] = target_transpose[d]; + } + root_ptr[0] = 0; root_ptr[1] = entries.size; + result.level = build_level(description, &entries, stored_dims, 0, root_ptr, 1, true, values); + if (!strcmp(target_format, "custom")) { + cJSON *user = cJSON_CreateObject(); + if (bsp_write_tensor(argv[2], result, NULL, user, compression) != BSP_SUCCESS) + die("failed to write output tensor"); + cJSON_Delete(user); + } else if (write_predefined(argv[2], result, target_format, compression) != + BSP_SUCCESS) { + die("failed to write predefined output"); + } + + bsp_destroy_tensor_t(result); + free_entries(&entries); + if (source_is_tensor) bsp_destroy_tensor_t(tensor); else bsp_destroy_matrix_t(&matrix); + free(source_transpose); free(target_transpose); free(stored_dims); + cJSON_Delete(description); cJSON_Delete(source_header); cJSON_Delete(target_header); + return 0; +} From 94c43e71f07389b96d89c80702dd118f9dda3b00 Mon Sep 17 00:00:00 2001 From: Willow Ahrens Date: Thu, 27 Aug 2026 13:18:46 -0400 Subject: [PATCH 02/16] Squashed 'third_party/cnpy/' content from commit 4e8810b git-subtree-dir: third_party/cnpy git-subtree-split: 4e8810b1a8637695171ed346ce68f6984e585ef4 --- CMakeLists.txt | 30 +++++ LICENSE | 21 +++ README.md | 55 ++++++++ cnpy.cpp | 340 +++++++++++++++++++++++++++++++++++++++++++++++++ cnpy.h | 269 ++++++++++++++++++++++++++++++++++++++ example1.cpp | 55 ++++++++ mat2npz | 18 +++ npy2mat | 15 +++ npz2mat | 15 +++ 9 files changed, 818 insertions(+) create mode 100644 CMakeLists.txt create mode 100644 LICENSE create mode 100644 README.md create mode 100644 cnpy.cpp create mode 100644 cnpy.h create mode 100644 example1.cpp create mode 100644 mat2npz create mode 100644 npy2mat create mode 100755 npz2mat diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000..9eb550f --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,30 @@ +CMAKE_MINIMUM_REQUIRED(VERSION 3.0 FATAL_ERROR) +if(COMMAND cmake_policy) + cmake_policy(SET CMP0003 NEW) +endif(COMMAND cmake_policy) + +project(CNPY) + +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11") + +option(ENABLE_STATIC "Build static (.a) library" ON) + +find_package(ZLIB REQUIRED) + +include_directories(${ZLIB_INCLUDE_DIRS}) + +add_library(cnpy SHARED "cnpy.cpp") +target_link_libraries(cnpy ${ZLIB_LIBRARIES}) +install(TARGETS "cnpy" LIBRARY DESTINATION lib PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE) + +if(ENABLE_STATIC) + add_library(cnpy-static STATIC "cnpy.cpp") + set_target_properties(cnpy-static PROPERTIES OUTPUT_NAME "cnpy") + install(TARGETS "cnpy-static" ARCHIVE DESTINATION lib) +endif(ENABLE_STATIC) + +install(FILES "cnpy.h" DESTINATION include) +install(FILES "mat2npz" "npy2mat" "npz2mat" DESTINATION bin PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE) + +add_executable(example1 example1.cpp) +target_link_libraries(example1 cnpy) diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..e60eadb --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +The MIT License + +Copyright (c) Carl Rogers, 2011 + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..37c4a43 --- /dev/null +++ b/README.md @@ -0,0 +1,55 @@ +# Purpose: + +NumPy offers the `save` method for easy saving of arrays into .npy and `savez` for zipping multiple .npy arrays together into a .npz file. + +`cnpy` lets you read and write to these formats in C++. + +The motivation comes from scientific programming where large amounts of data are generated in C++ and analyzed in Python. + +Writing to .npy has the advantage of using low-level C++ I/O (fread and fwrite) for speed and binary format for size. +The .npy file header takes care of specifying the size, shape, and data type of the array, so specifying the format of the data is unnecessary. + +Loading data written in numpy formats into C++ is equally simple, but requires you to type-cast the loaded data to the type of your choice. + +# Installation: + +Default installation directory is /usr/local. +To specify a different directory, add `-DCMAKE_INSTALL_PREFIX=/path/to/install/dir` to the cmake invocation in step 4. + +1. get [cmake](www.cmake.org) +2. create a build directory, say $HOME/build +3. cd $HOME/build +4. cmake /path/to/cnpy +5. make +6. make install + +# Using: + +To use, `#include"cnpy.h"` in your source code. Compile the source code mycode.cpp as + +```bash +g++ -o mycode mycode.cpp -L/path/to/install/dir -lcnpy -lz --std=c++11 +``` + +# Description: + +There are two functions for writing data: `npy_save` and `npz_save`. + +There are 3 functions for reading: +- `npy_load` will load a .npy file. +- `npz_load(fname)` will load a .npz and return a dictionary of NpyArray structues. +- `npz_load(fname,varname)` will load and return the NpyArray for data varname from the specified .npz file. + +The data structure for loaded data is below. +Data is accessed via the `data()`-method, which returns a pointer of the specified type (which must match the underlying datatype of the data). +The array shape and word size are read from the npy header. + +```c++ +struct NpyArray { + std::vector shape; + size_t word_size; + template T* data(); +}; +``` + +See [example1.cpp](example1.cpp) for examples of how to use the library. example1 will also be build during cmake installation. diff --git a/cnpy.cpp b/cnpy.cpp new file mode 100644 index 0000000..2d28578 --- /dev/null +++ b/cnpy.cpp @@ -0,0 +1,340 @@ +//Copyright (C) 2011 Carl Rogers +//Released under MIT License +//license available in LICENSE file, or at http://www.opensource.org/licenses/mit-license.php + +#include"cnpy.h" +#include +#include +#include +#include +#include +#include +#include +#include + +char cnpy::BigEndianTest() { + int x = 1; + return (((char *)&x)[0]) ? '<' : '>'; +} + +char cnpy::map_type(const std::type_info& t) +{ + if(t == typeid(float) ) return 'f'; + if(t == typeid(double) ) return 'f'; + if(t == typeid(long double) ) return 'f'; + + if(t == typeid(int) ) return 'i'; + if(t == typeid(char) ) return 'i'; + if(t == typeid(short) ) return 'i'; + if(t == typeid(long) ) return 'i'; + if(t == typeid(long long) ) return 'i'; + + if(t == typeid(unsigned char) ) return 'u'; + if(t == typeid(unsigned short) ) return 'u'; + if(t == typeid(unsigned long) ) return 'u'; + if(t == typeid(unsigned long long) ) return 'u'; + if(t == typeid(unsigned int) ) return 'u'; + + if(t == typeid(bool) ) return 'b'; + + if(t == typeid(std::complex) ) return 'c'; + if(t == typeid(std::complex) ) return 'c'; + if(t == typeid(std::complex) ) return 'c'; + + else return '?'; +} + +template<> std::vector& cnpy::operator+=(std::vector& lhs, const std::string rhs) { + lhs.insert(lhs.end(),rhs.begin(),rhs.end()); + return lhs; +} + +template<> std::vector& cnpy::operator+=(std::vector& lhs, const char* rhs) { + //write in little endian + size_t len = strlen(rhs); + lhs.reserve(len); + for(size_t byte = 0; byte < len; byte++) { + lhs.push_back(rhs[byte]); + } + return lhs; +} + +void cnpy::parse_npy_header(unsigned char* buffer,size_t& word_size, std::vector& shape, bool& fortran_order) { + //std::string magic_string(buffer,6); + uint8_t major_version = *reinterpret_cast(buffer+6); + uint8_t minor_version = *reinterpret_cast(buffer+7); + uint16_t header_len = *reinterpret_cast(buffer+8); + std::string header(reinterpret_cast(buffer+9),header_len); + + size_t loc1, loc2; + + //fortran order + loc1 = header.find("fortran_order")+16; + fortran_order = (header.substr(loc1,4) == "True" ? true : false); + + //shape + loc1 = header.find("("); + loc2 = header.find(")"); + + std::regex num_regex("[0-9][0-9]*"); + std::smatch sm; + shape.clear(); + + std::string str_shape = header.substr(loc1+1,loc2-loc1-1); + while(std::regex_search(str_shape, sm, num_regex)) { + shape.push_back(std::stoi(sm[0].str())); + str_shape = sm.suffix().str(); + } + + //endian, word size, data type + //byte order code | stands for not applicable. + //not sure when this applies except for byte array + loc1 = header.find("descr")+9; + bool littleEndian = (header[loc1] == '<' || header[loc1] == '|' ? true : false); + assert(littleEndian); + + //char type = header[loc1+1]; + //assert(type == map_type(T)); + + std::string str_ws = header.substr(loc1+2); + loc2 = str_ws.find("'"); + word_size = atoi(str_ws.substr(0,loc2).c_str()); +} + +void cnpy::parse_npy_header(FILE* fp, size_t& word_size, std::vector& shape, bool& fortran_order) { + char buffer[256]; + size_t res = fread(buffer,sizeof(char),11,fp); + if(res != 11) + throw std::runtime_error("parse_npy_header: failed fread"); + std::string header = fgets(buffer,256,fp); + assert(header[header.size()-1] == '\n'); + + size_t loc1, loc2; + + //fortran order + loc1 = header.find("fortran_order"); + if (loc1 == std::string::npos) + throw std::runtime_error("parse_npy_header: failed to find header keyword: 'fortran_order'"); + loc1 += 16; + fortran_order = (header.substr(loc1,4) == "True" ? true : false); + + //shape + loc1 = header.find("("); + loc2 = header.find(")"); + if (loc1 == std::string::npos || loc2 == std::string::npos) + throw std::runtime_error("parse_npy_header: failed to find header keyword: '(' or ')'"); + + std::regex num_regex("[0-9][0-9]*"); + std::smatch sm; + shape.clear(); + + std::string str_shape = header.substr(loc1+1,loc2-loc1-1); + while(std::regex_search(str_shape, sm, num_regex)) { + shape.push_back(std::stoi(sm[0].str())); + str_shape = sm.suffix().str(); + } + + //endian, word size, data type + //byte order code | stands for not applicable. + //not sure when this applies except for byte array + loc1 = header.find("descr"); + if (loc1 == std::string::npos) + throw std::runtime_error("parse_npy_header: failed to find header keyword: 'descr'"); + loc1 += 9; + bool littleEndian = (header[loc1] == '<' || header[loc1] == '|' ? true : false); + assert(littleEndian); + + //char type = header[loc1+1]; + //assert(type == map_type(T)); + + std::string str_ws = header.substr(loc1+2); + loc2 = str_ws.find("'"); + word_size = atoi(str_ws.substr(0,loc2).c_str()); +} + +void cnpy::parse_zip_footer(FILE* fp, uint16_t& nrecs, size_t& global_header_size, size_t& global_header_offset) +{ + std::vector footer(22); + fseek(fp,-22,SEEK_END); + size_t res = fread(&footer[0],sizeof(char),22,fp); + if(res != 22) + throw std::runtime_error("parse_zip_footer: failed fread"); + + uint16_t disk_no, disk_start, nrecs_on_disk, comment_len; + disk_no = *(uint16_t*) &footer[4]; + disk_start = *(uint16_t*) &footer[6]; + nrecs_on_disk = *(uint16_t*) &footer[8]; + nrecs = *(uint16_t*) &footer[10]; + global_header_size = *(uint32_t*) &footer[12]; + global_header_offset = *(uint32_t*) &footer[16]; + comment_len = *(uint16_t*) &footer[20]; + + assert(disk_no == 0); + assert(disk_start == 0); + assert(nrecs_on_disk == nrecs); + assert(comment_len == 0); +} + +cnpy::NpyArray load_the_npy_file(FILE* fp) { + std::vector shape; + size_t word_size; + bool fortran_order; + cnpy::parse_npy_header(fp,word_size,shape,fortran_order); + + cnpy::NpyArray arr(shape, word_size, fortran_order); + size_t nread = fread(arr.data(),1,arr.num_bytes(),fp); + if(nread != arr.num_bytes()) + throw std::runtime_error("load_the_npy_file: failed fread"); + return arr; +} + +cnpy::NpyArray load_the_npz_array(FILE* fp, uint32_t compr_bytes, uint32_t uncompr_bytes) { + + std::vector buffer_compr(compr_bytes); + std::vector buffer_uncompr(uncompr_bytes); + size_t nread = fread(&buffer_compr[0],1,compr_bytes,fp); + if(nread != compr_bytes) + throw std::runtime_error("load_the_npy_file: failed fread"); + + int err; + z_stream d_stream; + + d_stream.zalloc = Z_NULL; + d_stream.zfree = Z_NULL; + d_stream.opaque = Z_NULL; + d_stream.avail_in = 0; + d_stream.next_in = Z_NULL; + err = inflateInit2(&d_stream, -MAX_WBITS); + + d_stream.avail_in = compr_bytes; + d_stream.next_in = &buffer_compr[0]; + d_stream.avail_out = uncompr_bytes; + d_stream.next_out = &buffer_uncompr[0]; + + err = inflate(&d_stream, Z_FINISH); + err = inflateEnd(&d_stream); + + std::vector shape; + size_t word_size; + bool fortran_order; + cnpy::parse_npy_header(&buffer_uncompr[0],word_size,shape,fortran_order); + + cnpy::NpyArray array(shape, word_size, fortran_order); + + size_t offset = uncompr_bytes - array.num_bytes(); + memcpy(array.data(),&buffer_uncompr[0]+offset,array.num_bytes()); + + return array; +} + +cnpy::npz_t cnpy::npz_load(std::string fname) { + FILE* fp = fopen(fname.c_str(),"rb"); + + if(!fp) { + throw std::runtime_error("npz_load: Error! Unable to open file "+fname+"!"); + } + + cnpy::npz_t arrays; + + while(1) { + std::vector local_header(30); + size_t headerres = fread(&local_header[0],sizeof(char),30,fp); + if(headerres != 30) + throw std::runtime_error("npz_load: failed fread"); + + //if we've reached the global header, stop reading + if(local_header[2] != 0x03 || local_header[3] != 0x04) break; + + //read in the variable name + uint16_t name_len = *(uint16_t*) &local_header[26]; + std::string varname(name_len,' '); + size_t vname_res = fread(&varname[0],sizeof(char),name_len,fp); + if(vname_res != name_len) + throw std::runtime_error("npz_load: failed fread"); + + //erase the lagging .npy + varname.erase(varname.end()-4,varname.end()); + + //read in the extra field + uint16_t extra_field_len = *(uint16_t*) &local_header[28]; + if(extra_field_len > 0) { + std::vector buff(extra_field_len); + size_t efield_res = fread(&buff[0],sizeof(char),extra_field_len,fp); + if(efield_res != extra_field_len) + throw std::runtime_error("npz_load: failed fread"); + } + + uint16_t compr_method = *reinterpret_cast(&local_header[0]+8); + uint32_t compr_bytes = *reinterpret_cast(&local_header[0]+18); + uint32_t uncompr_bytes = *reinterpret_cast(&local_header[0]+22); + + if(compr_method == 0) {arrays[varname] = load_the_npy_file(fp);} + else {arrays[varname] = load_the_npz_array(fp,compr_bytes,uncompr_bytes);} + } + + fclose(fp); + return arrays; +} + +cnpy::NpyArray cnpy::npz_load(std::string fname, std::string varname) { + FILE* fp = fopen(fname.c_str(),"rb"); + + if(!fp) throw std::runtime_error("npz_load: Unable to open file "+fname); + + while(1) { + std::vector local_header(30); + size_t header_res = fread(&local_header[0],sizeof(char),30,fp); + if(header_res != 30) + throw std::runtime_error("npz_load: failed fread"); + + //if we've reached the global header, stop reading + if(local_header[2] != 0x03 || local_header[3] != 0x04) break; + + //read in the variable name + uint16_t name_len = *(uint16_t*) &local_header[26]; + std::string vname(name_len,' '); + size_t vname_res = fread(&vname[0],sizeof(char),name_len,fp); + if(vname_res != name_len) + throw std::runtime_error("npz_load: failed fread"); + vname.erase(vname.end()-4,vname.end()); //erase the lagging .npy + + //read in the extra field + uint16_t extra_field_len = *(uint16_t*) &local_header[28]; + fseek(fp,extra_field_len,SEEK_CUR); //skip past the extra field + + uint16_t compr_method = *reinterpret_cast(&local_header[0]+8); + uint32_t compr_bytes = *reinterpret_cast(&local_header[0]+18); + uint32_t uncompr_bytes = *reinterpret_cast(&local_header[0]+22); + + if(vname == varname) { + NpyArray array = (compr_method == 0) ? load_the_npy_file(fp) : load_the_npz_array(fp,compr_bytes,uncompr_bytes); + fclose(fp); + return array; + } + else { + //skip past the data + uint32_t size = *(uint32_t*) &local_header[22]; + fseek(fp,size,SEEK_CUR); + } + } + + fclose(fp); + + //if we get here, we haven't found the variable in the file + throw std::runtime_error("npz_load: Variable name "+varname+" not found in "+fname); +} + +cnpy::NpyArray cnpy::npy_load(std::string fname) { + + FILE* fp = fopen(fname.c_str(), "rb"); + + if(!fp) throw std::runtime_error("npy_load: Unable to open file "+fname); + + NpyArray arr = load_the_npy_file(fp); + + fclose(fp); + return arr; +} + + + diff --git a/cnpy.h b/cnpy.h new file mode 100644 index 0000000..0d3bb4c --- /dev/null +++ b/cnpy.h @@ -0,0 +1,269 @@ +//Copyright (C) 2011 Carl Rogers +//Released under MIT License +//license available in LICENSE file, or at http://www.opensource.org/licenses/mit-license.php + +#ifndef LIBCNPY_H_ +#define LIBCNPY_H_ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace cnpy { + + struct NpyArray { + NpyArray(const std::vector& _shape, size_t _word_size, bool _fortran_order) : + shape(_shape), word_size(_word_size), fortran_order(_fortran_order) + { + num_vals = 1; + for(size_t i = 0;i < shape.size();i++) num_vals *= shape[i]; + data_holder = std::shared_ptr>( + new std::vector(num_vals * word_size)); + } + + NpyArray() : shape(0), word_size(0), fortran_order(0), num_vals(0) { } + + template + T* data() { + return reinterpret_cast(&(*data_holder)[0]); + } + + template + const T* data() const { + return reinterpret_cast(&(*data_holder)[0]); + } + + template + std::vector as_vec() const { + const T* p = data(); + return std::vector(p, p+num_vals); + } + + size_t num_bytes() const { + return data_holder->size(); + } + + std::shared_ptr> data_holder; + std::vector shape; + size_t word_size; + bool fortran_order; + size_t num_vals; + }; + + using npz_t = std::map; + + char BigEndianTest(); + char map_type(const std::type_info& t); + template std::vector create_npy_header(const std::vector& shape); + void parse_npy_header(FILE* fp,size_t& word_size, std::vector& shape, bool& fortran_order); + void parse_npy_header(unsigned char* buffer,size_t& word_size, std::vector& shape, bool& fortran_order); + void parse_zip_footer(FILE* fp, uint16_t& nrecs, size_t& global_header_size, size_t& global_header_offset); + npz_t npz_load(std::string fname); + NpyArray npz_load(std::string fname, std::string varname); + NpyArray npy_load(std::string fname); + + template std::vector& operator+=(std::vector& lhs, const T rhs) { + //write in little endian + for(size_t byte = 0; byte < sizeof(T); byte++) { + char val = *((char*)&rhs+byte); + lhs.push_back(val); + } + return lhs; + } + + template<> std::vector& operator+=(std::vector& lhs, const std::string rhs); + template<> std::vector& operator+=(std::vector& lhs, const char* rhs); + + + template void npy_save(std::string fname, const T* data, const std::vector shape, std::string mode = "w") { + FILE* fp = NULL; + std::vector true_data_shape; //if appending, the shape of existing + new data + + if(mode == "a") fp = fopen(fname.c_str(),"r+b"); + + if(fp) { + //file exists. we need to append to it. read the header, modify the array size + size_t word_size; + bool fortran_order; + parse_npy_header(fp,word_size,true_data_shape,fortran_order); + assert(!fortran_order); + + if(word_size != sizeof(T)) { + std::cout<<"libnpy error: "< header = create_npy_header(true_data_shape); + size_t nels = std::accumulate(shape.begin(),shape.end(),1,std::multiplies()); + + fseek(fp,0,SEEK_SET); + fwrite(&header[0],sizeof(char),header.size(),fp); + fseek(fp,0,SEEK_END); + fwrite(data,sizeof(T),nels,fp); + fclose(fp); + } + + template void npz_save(std::string zipname, std::string fname, const T* data, const std::vector& shape, std::string mode = "w") + { + //first, append a .npy to the fname + fname += ".npy"; + + //now, on with the show + FILE* fp = NULL; + uint16_t nrecs = 0; + size_t global_header_offset = 0; + std::vector global_header; + + if(mode == "a") fp = fopen(zipname.c_str(),"r+b"); + + if(fp) { + //zip file exists. we need to add a new npy file to it. + //first read the footer. this gives us the offset and size of the global header + //then read and store the global header. + //below, we will write the the new data at the start of the global header then append the global header and footer below it + size_t global_header_size; + parse_zip_footer(fp,nrecs,global_header_size,global_header_offset); + fseek(fp,global_header_offset,SEEK_SET); + global_header.resize(global_header_size); + size_t res = fread(&global_header[0],sizeof(char),global_header_size,fp); + if(res != global_header_size){ + throw std::runtime_error("npz_save: header read error while adding to existing zip"); + } + fseek(fp,global_header_offset,SEEK_SET); + } + else { + fp = fopen(zipname.c_str(),"wb"); + } + + std::vector npy_header = create_npy_header(shape); + + size_t nels = std::accumulate(shape.begin(),shape.end(),1,std::multiplies()); + size_t nbytes = nels*sizeof(T) + npy_header.size(); + + //get the CRC of the data to be added + uint32_t crc = crc32(0L,(uint8_t*)&npy_header[0],npy_header.size()); + crc = crc32(crc,(uint8_t*)data,nels*sizeof(T)); + + //build the local header + std::vector local_header; + local_header += "PK"; //first part of sig + local_header += (uint16_t) 0x0403; //second part of sig + local_header += (uint16_t) 20; //min version to extract + local_header += (uint16_t) 0; //general purpose bit flag + local_header += (uint16_t) 0; //compression method + local_header += (uint16_t) 0; //file last mod time + local_header += (uint16_t) 0; //file last mod date + local_header += (uint32_t) crc; //crc + local_header += (uint32_t) nbytes; //compressed size + local_header += (uint32_t) nbytes; //uncompressed size + local_header += (uint16_t) fname.size(); //fname length + local_header += (uint16_t) 0; //extra field length + local_header += fname; + + //build global header + global_header += "PK"; //first part of sig + global_header += (uint16_t) 0x0201; //second part of sig + global_header += (uint16_t) 20; //version made by + global_header.insert(global_header.end(),local_header.begin()+4,local_header.begin()+30); + global_header += (uint16_t) 0; //file comment length + global_header += (uint16_t) 0; //disk number where file starts + global_header += (uint16_t) 0; //internal file attributes + global_header += (uint32_t) 0; //external file attributes + global_header += (uint32_t) global_header_offset; //relative offset of local file header, since it begins where the global header used to begin + global_header += fname; + + //build footer + std::vector footer; + footer += "PK"; //first part of sig + footer += (uint16_t) 0x0605; //second part of sig + footer += (uint16_t) 0; //number of this disk + footer += (uint16_t) 0; //disk where footer starts + footer += (uint16_t) (nrecs+1); //number of records on this disk + footer += (uint16_t) (nrecs+1); //total number of records + footer += (uint32_t) global_header.size(); //nbytes of global headers + footer += (uint32_t) (global_header_offset + nbytes + local_header.size()); //offset of start of global headers, since global header now starts after newly written array + footer += (uint16_t) 0; //zip file comment length + + //write everything + fwrite(&local_header[0],sizeof(char),local_header.size(),fp); + fwrite(&npy_header[0],sizeof(char),npy_header.size(),fp); + fwrite(data,sizeof(T),nels,fp); + fwrite(&global_header[0],sizeof(char),global_header.size(),fp); + fwrite(&footer[0],sizeof(char),footer.size(),fp); + fclose(fp); + } + + template void npy_save(std::string fname, const std::vector data, std::string mode = "w") { + std::vector shape; + shape.push_back(data.size()); + npy_save(fname, &data[0], shape, mode); + } + + template void npz_save(std::string zipname, std::string fname, const std::vector data, std::string mode = "w") { + std::vector shape; + shape.push_back(data.size()); + npz_save(zipname, fname, &data[0], shape, mode); + } + + template std::vector create_npy_header(const std::vector& shape) { + + std::vector dict; + dict += "{'descr': '"; + dict += BigEndianTest(); + dict += map_type(typeid(T)); + dict += std::to_string(sizeof(T)); + dict += "', 'fortran_order': False, 'shape': ("; + dict += std::to_string(shape[0]); + for(size_t i = 1;i < shape.size();i++) { + dict += ", "; + dict += std::to_string(shape[i]); + } + if(shape.size() == 1) dict += ","; + dict += "), }"; + //pad with spaces so that preamble+dict is modulo 16 bytes. preamble is 10 bytes. dict needs to end with \n + int remainder = 16 - (10 + dict.size()) % 16; + dict.insert(dict.end(),remainder,' '); + dict.back() = '\n'; + + std::vector header; + header += (char) 0x93; + header += "NUMPY"; + header += (char) 0x01; //major version of numpy format + header += (char) 0x00; //minor version of numpy format + header += (uint16_t) dict.size(); + header.insert(header.end(),dict.begin(),dict.end()); + + return header; + } + + +} + +#endif diff --git a/example1.cpp b/example1.cpp new file mode 100644 index 0000000..70ac5aa --- /dev/null +++ b/example1.cpp @@ -0,0 +1,55 @@ +#include"cnpy.h" +#include +#include +#include +#include +#include + +const int Nx = 128; +const int Ny = 64; +const int Nz = 32; + +int main() +{ + //set random seed so that result is reproducible (for testing) + srand(0); + //create random data + std::vector> data(Nx*Ny*Nz); + for(int i = 0;i < Nx*Ny*Nz;i++) data[i] = std::complex(rand(),rand()); + + //save it to file + cnpy::npy_save("arr1.npy",&data[0],{Nz,Ny,Nx},"w"); + + //load it into a new array + cnpy::NpyArray arr = cnpy::npy_load("arr1.npy"); + std::complex* loaded_data = arr.data>(); + + //make sure the loaded data matches the saved data + assert(arr.word_size == sizeof(std::complex)); + assert(arr.shape.size() == 3 && arr.shape[0] == Nz && arr.shape[1] == Ny && arr.shape[2] == Nx); + for(int i = 0; i < Nx*Ny*Nz;i++) assert(data[i] == loaded_data[i]); + + //append the same data to file + //npy array on file now has shape (Nz+Nz,Ny,Nx) + cnpy::npy_save("arr1.npy",&data[0],{Nz,Ny,Nx},"a"); + + //now write to an npz file + //non-array variables are treated as 1D arrays with 1 element + double myVar1 = 1.2; + char myVar2 = 'a'; + cnpy::npz_save("out.npz","myVar1",&myVar1,{1},"w"); //"w" overwrites any existing file + cnpy::npz_save("out.npz","myVar2",&myVar2,{1},"a"); //"a" appends to the file we created above + cnpy::npz_save("out.npz","arr1",&data[0],{Nz,Ny,Nx},"a"); //"a" appends to the file we created above + + //load a single var from the npz file + cnpy::NpyArray arr2 = cnpy::npz_load("out.npz","arr1"); + + //load the entire npz file + cnpy::npz_t my_npz = cnpy::npz_load("out.npz"); + + //check that the loaded myVar1 matches myVar1 + cnpy::NpyArray arr_mv1 = my_npz["myVar1"]; + double* mv1 = arr_mv1.data(); + assert(arr_mv1.shape.size() == 1 && arr_mv1.shape[0] == 1); + assert(mv1[0] == myVar1); +} diff --git a/mat2npz b/mat2npz new file mode 100644 index 0000000..c1bbd3e --- /dev/null +++ b/mat2npz @@ -0,0 +1,18 @@ +#!/usr/bin/env python + +import sys +from numpy import savez +from scipy.io import loadmat + +assert len(sys.argv) > 1 + +files = sys.argv[1:] + +for f in files: + mat_vars = loadmat(f) + mat_vars.pop('__version__') + mat_vars.pop('__header__') + mat_vars.pop('__globals__') + + fn = f.replace('.mat','.npz') + savez(fn,**mat_vars) diff --git a/npy2mat b/npy2mat new file mode 100644 index 0000000..fdd5da7 --- /dev/null +++ b/npy2mat @@ -0,0 +1,15 @@ +#!/usr/bin/env python + +import sys +from numpy import load +from scipy.io import savemat + +assert len(sys.argv) > 1 + +files = sys.argv[1:] + +for f in files: + data = load(f) + fn = f.replace('.npy','') + fn = fn.replace('.','_') + savemat(fn,{fn : data}) diff --git a/npz2mat b/npz2mat new file mode 100755 index 0000000..2e317d6 --- /dev/null +++ b/npz2mat @@ -0,0 +1,15 @@ +#!/usr/bin/env python + +import sys +from numpy import load +from scipy.io import savemat + +assert len(sys.argv) > 1 + +files = sys.argv[1:] + +for f in files: + data = load(f) + fn = f.replace('.npz','') + fn = fn.replace('.','_') #matlab cant handle dots + savemat(fn,data) From fc50d94b0e89dc50689d5f7bc011fa5d2c74740b Mon Sep 17 00:00:00 2001 From: Willow Ahrens Date: Thu, 27 Aug 2026 13:42:33 -0400 Subject: [PATCH 03/16] add tests --- CMakeLists.txt | 2 + test/compliance/CMakeLists.txt | 27 +++ test/compliance/binsparse_to_binsparse.c | 22 +++ test/compliance/binsparse_to_npy.c | 115 +++++++++++ test/compliance/cnpy_c.cpp | 66 +++++++ test/compliance/cnpy_c.h | 25 +++ test/compliance/npy_to_binsparse.c | 142 ++++++++++++++ {compliance => test/compliance}/reformat.c | 213 ++++++++++++++------- test/compliance/reformat.h | 55 ++++++ test/compliance/run-binsparse-tests.sh | 61 ++++++ third_party/cnpy/cnpy.cpp | 2 +- third_party/cnpy/cnpy.h | 10 +- 12 files changed, 670 insertions(+), 70 deletions(-) create mode 100644 test/compliance/CMakeLists.txt create mode 100644 test/compliance/binsparse_to_binsparse.c create mode 100644 test/compliance/binsparse_to_npy.c create mode 100644 test/compliance/cnpy_c.cpp create mode 100644 test/compliance/cnpy_c.h create mode 100644 test/compliance/npy_to_binsparse.c rename {compliance => test/compliance}/reformat.c (77%) create mode 100644 test/compliance/reformat.h create mode 100755 test/compliance/run-binsparse-tests.sh diff --git a/CMakeLists.txt b/CMakeLists.txt index 1776473..0106346 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -122,6 +122,8 @@ install(FILES if(CMAKE_PROJECT_NAME STREQUAL PROJECT_NAME) enable_testing() + add_subdirectory(test/compliance) + if (ENABLE_SANITIZERS) set(SANITIZER_FLAGS "-fsanitize=address,undefined") target_compile_options(binsparse INTERFACE ${SANITIZER_FLAGS} -g -O1 -fno-omit-frame-pointer) diff --git a/test/compliance/CMakeLists.txt b/test/compliance/CMakeLists.txt new file mode 100644 index 0000000..0ac3c0e --- /dev/null +++ b/test/compliance/CMakeLists.txt @@ -0,0 +1,27 @@ +find_package(ZLIB REQUIRED) + +add_library(cnpy_vendor STATIC + ${PROJECT_SOURCE_DIR}/third_party/cnpy/cnpy.cpp + cnpy_c.cpp +) +target_include_directories(cnpy_vendor PUBLIC + ${PROJECT_SOURCE_DIR}/third_party/cnpy + ${CMAKE_CURRENT_SOURCE_DIR} +) +target_link_libraries(cnpy_vendor PUBLIC ZLIB::ZLIB) +target_compile_features(cnpy_vendor PUBLIC cxx_std_11) + +add_library(compliance_core STATIC reformat.c) +target_include_directories(compliance_core PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) +target_link_libraries(compliance_core PUBLIC binsparse) + +add_executable(npy_to_binsparse npy_to_binsparse.c) +target_link_libraries(npy_to_binsparse PRIVATE compliance_core cnpy_vendor) +set_target_properties(npy_to_binsparse PROPERTIES LINKER_LANGUAGE CXX) + +add_executable(binsparse_to_npy binsparse_to_npy.c) +target_link_libraries(binsparse_to_npy PRIVATE compliance_core cnpy_vendor) +set_target_properties(binsparse_to_npy PROPERTIES LINKER_LANGUAGE CXX) + +add_executable(binsparse_to_binsparse binsparse_to_binsparse.c) +target_link_libraries(binsparse_to_binsparse PRIVATE compliance_core) diff --git a/test/compliance/binsparse_to_binsparse.c b/test/compliance/binsparse_to_binsparse.c new file mode 100644 index 0000000..df71dea --- /dev/null +++ b/test/compliance/binsparse_to_binsparse.c @@ -0,0 +1,22 @@ +#include "reformat.h" + +#include + +int main(int argc, char **argv) { + cJSON *header; + bsp_error_t error; + + if (argc != 3) { + fprintf(stderr, "usage: binsparse_to_binsparse \n"); + return 2; + } + header = input_header(argv[1]); + error = bsp_reformat_file(argv[1], argv[2], header, 0); + cJSON_Delete(header); + if (error != BSP_SUCCESS) { + fprintf(stderr, "binsparse_to_binsparse: %s\n", + bsp_get_error_string(error)); + return 1; + } + return 0; +} diff --git a/test/compliance/binsparse_to_npy.c b/test/compliance/binsparse_to_npy.c new file mode 100644 index 0000000..448fd55 --- /dev/null +++ b/test/compliance/binsparse_to_npy.c @@ -0,0 +1,115 @@ +#include "reformat.h" +#include "cnpy_c.h" + +#include +#include +#include +#include +#include + +static size_t shape_size(const size_t *shape, int rank) { + size_t size = 1; + for (int d = 0; d < rank; ++d) size *= shape[d]; + return size; +} + +int main(int argc, char **argv) { + cJSON *header; + const char *format; + bool custom; + bsp_tensor_t tensor = bsp_construct_default_tensor_t(); + bsp_matrix_t matrix; + bsp_array_t values, fill_value; + entries_t entries; + size_t *shape, *transpose, size; + void *dense; + uint8_t *pattern; + bool owns_fill = false; + + if (argc != 5) { + fprintf(stderr, "usage: binsparse_to_npy " + " \n"); + return 2; + } + header = input_header(argv[1]); + format = cJSON_GetStringValue(cJSON_GetObjectItemCaseSensitive(header, "format")); + if (format == NULL) die("input header has no format"); + custom = !strcmp(format, "custom"); + int rank; + if (custom) { + tensor = bsp_read_tensor(argv[1], NULL); + tensor.is_iso = header_values_are_iso(header); + rank = tensor.rank; + shape = tensor.dims; + values = bsp_get_tensor_values(tensor); + entries = tensor_to_coo(&tensor); + } else { + if (bsp_read_matrix(&matrix, argv[1], NULL) != BSP_SUCCESS) + die("cannot read Binsparse matrix"); + rank = bsp_matrix_format_is_vector(matrix.format) ? 1 : 2; + shape = malloc((size_t) rank * sizeof(size_t)); + shape[0] = matrix.nrows; + if (rank == 2) shape[1] = matrix.ncols; + values = matrix.values; + entries = matrix_to_coo(&matrix); + } + transpose = rank ? malloc((size_t) rank * sizeof(size_t)) : NULL; + for (int d = 0; d < rank; ++d) transpose[d] = (size_t) d; + if (custom && tensor.transpose) + memcpy(transpose, tensor.transpose, (size_t) rank * sizeof(size_t)); + else if (!custom && rank == 2 && + (matrix.format == BSP_DMATC || matrix.format == BSP_CSC || + matrix.format == BSP_DCSC || matrix.format == BSP_COOC)) { + transpose[0] = 1; transpose[1] = 0; + } + + bsp_construct_default_array_t(&fill_value); + cJSON *fill = cJSON_GetObjectItemCaseSensitive(header, "fill"); + if (cJSON_IsTrue(fill)) { + hid_t file = H5Fopen(argv[1], H5F_ACC_RDONLY, H5P_DEFAULT); + if (file < 0 || bsp_read_array(&fill_value, file, "fill_value") != BSP_SUCCESS) + die("cannot read fill_value"); + H5Fclose(file); + const char *fill_type = cJSON_GetStringValue(cJSON_GetObjectItemCaseSensitive( + cJSON_GetObjectItemCaseSensitive(header, "data_types"), "fill_value")); + if (fill_type != NULL && strncmp(fill_type, "complex[", 8) == 0 && + bsp_fp_array_to_complex(&fill_value) != BSP_SUCCESS) + die("invalid complex fill_value"); + owns_fill = true; + } else { + if (bsp_construct_array_t(&fill_value, 1, values.type) != BSP_SUCCESS) + die("out of memory"); + memset(fill_value.data, 0, bsp_type_size(values.type)); + owns_fill = true; + } + size = shape_size(shape, rank); + dense = malloc(size * bsp_type_size(values.type)); + pattern = calloc(size, 1); + if ((size && !dense) || (size && !pattern)) die("out of memory"); + for (size_t i = 0; i < size; ++i) + memcpy((char *) dense + i * bsp_type_size(values.type), fill_value.data, + bsp_type_size(values.type)); + for (size_t k = 0; k < entries.size; ++k) { + size_t linear = 0; + for (int logical = 0; logical < rank; ++logical) { + int stored = 0; + while (stored < rank && transpose[stored] != (size_t) logical) ++stored; + linear = linear * shape[logical] + entries.entry[k].coord[stored]; + } + pattern[linear] = 1; + memcpy((char *) dense + linear * bsp_type_size(values.type), + (char *) values.data + entries.entry[k].value * bsp_type_size(values.type), + bsp_type_size(values.type)); + } + if (cnpy_c_save(argv[2], dense, shape, (size_t) rank, values.type) || + cnpy_c_save(argv[3], pattern, shape, (size_t) rank, BSP_BINT8) || + cnpy_c_save(argv[4], fill_value.data, NULL, 0, fill_value.type)) + die("cannot write NPY output"); + + free(dense); free(pattern); free(transpose); free_entries(&entries); + if (owns_fill) bsp_destroy_array_t(&fill_value); + if (custom) bsp_destroy_tensor_t(tensor); + else { free(shape); bsp_destroy_matrix_t(&matrix); } + cJSON_Delete(header); + return 0; +} diff --git a/test/compliance/cnpy_c.cpp b/test/compliance/cnpy_c.cpp new file mode 100644 index 0000000..8354aeb --- /dev/null +++ b/test/compliance/cnpy_c.cpp @@ -0,0 +1,66 @@ +#include "cnpy_c.h" + +#include + +#include +#include +#include +#include +#include + +extern "C" int cnpy_c_load(const char *path, cnpy_c_array *out) { + try { + cnpy::NpyArray array = cnpy::npy_load(path); + out->size = array.num_vals; + out->word_size = array.word_size; + out->rank = array.shape.size(); + out->fortran_order = array.fortran_order; + out->shape = static_cast(malloc(out->rank * sizeof(size_t))); + out->data = malloc(array.num_bytes()); + if ((out->rank && !out->shape) || (array.num_bytes() && !out->data)) return 1; + if (out->rank) memcpy(out->shape, array.shape.data(), out->rank * sizeof(size_t)); + if (array.num_bytes()) memcpy(out->data, array.data(), array.num_bytes()); + return 0; + } catch (const std::exception &) { + return 1; + } +} + +extern "C" void cnpy_c_destroy(cnpy_c_array *array) { + free(array->data); + free(array->shape); + memset(array, 0, sizeof(*array)); +} + +template +static void save(const char *path, const void *data, const size_t *shape, + size_t rank) { + std::vector dimensions; + if (rank != 0) dimensions.assign(shape, shape + rank); + cnpy::npy_save(path, static_cast(data), dimensions); +} + +extern "C" int cnpy_c_save(const char *path, const void *data, + const size_t *shape, size_t rank, int type) { + try { + switch (type) { + case 0: save(path, data, shape, rank); break; + case 1: save(path, data, shape, rank); break; + case 2: save(path, data, shape, rank); break; + case 3: save(path, data, shape, rank); break; + case 4: save(path, data, shape, rank); break; + case 5: save(path, data, shape, rank); break; + case 6: save(path, data, shape, rank); break; + case 7: save(path, data, shape, rank); break; + case 8: save(path, data, shape, rank); break; + case 9: save(path, data, shape, rank); break; + case 10: save(path, data, shape, rank); break; + case 11: save>(path, data, shape, rank); break; + case 12: save>(path, data, shape, rank); break; + default: return 1; + } + return 0; + } catch (const std::exception &) { + return 1; + } +} diff --git a/test/compliance/cnpy_c.h b/test/compliance/cnpy_c.h new file mode 100644 index 0000000..bb1b2d7 --- /dev/null +++ b/test/compliance/cnpy_c.h @@ -0,0 +1,25 @@ +#pragma once + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct { + void *data; + size_t size; + size_t word_size; + size_t rank; + size_t *shape; + int fortran_order; +} cnpy_c_array; + +int cnpy_c_load(const char *path, cnpy_c_array *array); +void cnpy_c_destroy(cnpy_c_array *array); +int cnpy_c_save(const char *path, const void *data, const size_t *shape, + size_t rank, int bsp_type); + +#ifdef __cplusplus +} +#endif diff --git a/test/compliance/npy_to_binsparse.c b/test/compliance/npy_to_binsparse.c new file mode 100644 index 0000000..0b858cc --- /dev/null +++ b/test/compliance/npy_to_binsparse.c @@ -0,0 +1,142 @@ +#include "reformat.h" +#include "cnpy_c.h" + +#include +#include +#include +#include + +static size_t product(const size_t *shape, size_t rank) { + size_t size = 1; + for (size_t d = 0; d < rank; ++d) size *= shape[d]; + return size; +} + +int main(int argc, char **argv) { + cnpy_c_array dense = {0}, pattern = {0}, fill = {0}; + cJSON *header, *shape_json, *custom, *description; + const char *format, *value_name; + bsp_type_t value_type; + bsp_array_t values; + bsp_tensor_t result = bsp_construct_default_tensor_t(); + entries_t entries = {0}; + reformat_context_t context; + size_t *logical_coord, *source_transpose, *target_transpose, *stored_dims, + root_ptr[2]; + size_t stored = 0; + bool iso; + + if (argc != 6) { + fprintf(stderr, "usage: npy_to_binsparse " + " \n"); + return 2; + } + if (cnpy_c_load(argv[1], &dense) || cnpy_c_load(argv[2], &pattern) || + cnpy_c_load(argv[3], &fill)) + die("cannot read NPY input"); + if (dense.fortran_order || pattern.fortran_order || fill.fortran_order) + die("Fortran-order NPY arrays are unsupported"); + if (dense.rank != pattern.rank || dense.size != pattern.size || + pattern.word_size != 1 || fill.size != 1) + die("incompatible tensor, pattern, or fill-value NPY input"); + + header = read_json(argv[4]); + shape_json = cJSON_GetObjectItemCaseSensitive(header, "shape"); + result.rank = cJSON_GetArraySize(shape_json); + if ((size_t) result.rank != dense.rank) die("header shape rank mismatch"); + result.dims = result.rank ? malloc((size_t) result.rank * sizeof(size_t)) : NULL; + result.transpose = result.rank ? malloc((size_t) result.rank * sizeof(size_t)) : NULL; + logical_coord = result.rank ? calloc((size_t) result.rank, sizeof(size_t)) : NULL; + target_transpose = result.rank ? malloc((size_t) result.rank * sizeof(size_t)) : NULL; + source_transpose = result.rank ? malloc((size_t) result.rank * sizeof(size_t)) : NULL; + stored_dims = result.rank ? malloc((size_t) result.rank * sizeof(size_t)) : NULL; + entries.rank = result.rank; + for (int d = 0; d < result.rank; ++d) { + result.dims[d] = (size_t) cJSON_GetNumberValue(cJSON_GetArrayItem(shape_json, d)); + if (result.dims[d] != dense.shape[d]) die("header shape mismatch"); + source_transpose[d] = result.transpose[d] = target_transpose[d] = (size_t) d; + } + if (product(result.dims, (size_t) result.rank) != dense.size) + die("NPY element count does not match shape"); + + context.data_types = cJSON_GetObjectItemCaseSensitive(header, "data_types"); + value_name = cJSON_GetStringValue( + cJSON_GetObjectItemCaseSensitive(context.data_types, "values")); + value_type = type_from_name(value_name); + if (value_type == BSP_INVALID_TYPE || bsp_type_size(value_type) != dense.word_size) + die("NPY dtype does not match header values type"); + if (fill.word_size != bsp_type_size(value_type)) + die("fill-value dtype does not match header values type"); + if (!cJSON_HasObjectItem(header, "fill")) + cJSON_AddBoolToObject(header, "fill", true); + if (!cJSON_HasObjectItem(context.data_types, "fill_value")) + cJSON_AddStringToObject(context.data_types, "fill_value", + bsp_get_type_string(value_type)); + iso = value_name != NULL && !strncmp(value_name, "iso[", 4); + for (size_t i = 0; i < pattern.size; ++i) + if (((uint8_t *) pattern.data)[i]) ++stored; + if (bsp_construct_array_t(&values, iso && stored ? 1 : stored, value_type) != + BSP_SUCCESS) + die("out of memory"); + + size_t value_index = 0; + for (size_t linear = 0; linear < dense.size; ++linear) { + if (((uint8_t *) pattern.data)[linear]) { + push_entry(&entries, logical_coord, iso ? 0 : value_index); + if (!iso || value_index == 0) + memcpy((char *) values.data + (iso ? 0 : value_index) * dense.word_size, + (char *) dense.data + linear * dense.word_size, dense.word_size); + ++value_index; + } + for (int d = result.rank - 1; d >= 0; --d) { + if (++logical_coord[d] < result.dims[d]) break; + logical_coord[d] = 0; + } + } + result.nnz = stored; + result.is_iso = iso; + + format = cJSON_GetStringValue(cJSON_GetObjectItemCaseSensitive(header, "format")); + if (format == NULL) die("header has no format"); + if (!strcmp(format, "custom")) { + custom = cJSON_GetObjectItemCaseSensitive(header, "custom"); + description = cJSON_Duplicate( + cJSON_GetObjectItemCaseSensitive(custom, "level"), 1); + if (description == NULL) die("custom target requires custom.level"); + cJSON *transpose = cJSON_GetObjectItemCaseSensitive(custom, "transpose"); + if (transpose != NULL) + for (int d = 0; d < result.rank; ++d) + result.transpose[d] = target_transpose[d] = + (size_t) cJSON_GetNumberValue(cJSON_GetArrayItem(transpose, d)); + } else { + description = predefined_level(format, target_transpose, result.rank); + if (description == NULL) die("unknown or rank-incompatible target format"); + for (int d = 0; d < result.rank; ++d) result.transpose[d] = target_transpose[d]; + } + apply_transpose(&entries, source_transpose, target_transpose); + for (int d = 0; d < result.rank; ++d) stored_dims[d] = result.dims[target_transpose[d]]; + root_ptr[0] = 0; root_ptr[1] = stored; + result.level = build_level(&context, description, &entries, stored_dims, 0, + root_ptr, 1, true, values); + if (!strcmp(format, "custom")) { + cJSON *user = cJSON_CreateObject(); + if (bsp_write_tensor(argv[5], result, NULL, user, 0) != BSP_SUCCESS) + die("cannot write Binsparse tensor"); + cJSON_Delete(user); + } else if (write_predefined(argv[5], result, format, 0) != BSP_SUCCESS) { + die("cannot write Binsparse tensor"); + } + bsp_array_t fill_array; + fill_array.data = fill.data; + fill_array.size = 1; + fill_array.type = value_type; + fill_array.allocator = bsp_default_allocator; + complete_output(argv[5], header, &fill_array); + + bsp_destroy_tensor_t(result); + free_entries(&entries); + free(logical_coord); free(source_transpose); free(target_transpose); free(stored_dims); + cJSON_Delete(description); cJSON_Delete(header); + cnpy_c_destroy(&dense); cnpy_c_destroy(&pattern); cnpy_c_destroy(&fill); + return 0; +} diff --git a/compliance/reformat.c b/test/compliance/reformat.c similarity index 77% rename from compliance/reformat.c rename to test/compliance/reformat.c index 3b8862c..bad1a30 100644 --- a/compliance/reformat.c +++ b/test/compliance/reformat.c @@ -12,6 +12,9 @@ * custom dense/sparse level trees are accepted. */ +#include "reformat.h" + +#include #include #include #include @@ -26,26 +29,37 @@ #include #include -bsp_error_t bsp_read_attribute(char **attribute, hid_t f, char *name); - -typedef struct { - size_t *coord; - size_t value; -} entry_t; +bsp_type_t type_from_name(const char *name) { + size_t length; + char base[32]; + if (name == NULL) return BSP_INVALID_TYPE; + length = strlen(name); + if (length > 5 && !strncmp(name, "iso[", 4) && name[length - 1] == ']') { + if (length - 5 >= sizeof(base)) return BSP_INVALID_TYPE; + memcpy(base, name + 4, length - 5); + base[length - 5] = '\0'; + name = base; + } + for (int type = BSP_UINT8; type < BSP_INVALID_TYPE; ++type) + if (!strcmp(name, bsp_get_type_string((bsp_type_t) type))) + return (bsp_type_t) type; + return BSP_INVALID_TYPE; +} -typedef struct { - entry_t *entry; - size_t size; - size_t capacity; - int rank; -} entries_t; +static bsp_type_t requested_type(const reformat_context_t *context, + const char *key, bsp_type_t fallback) { + const char *name = cJSON_GetStringValue( + cJSON_GetObjectItemCaseSensitive(context->data_types, key)); + bsp_type_t type = type_from_name(name); + return type == BSP_INVALID_TYPE ? fallback : type; +} -static void die(const char *message) { +void die(const char *message) { fprintf(stderr, "reformat: %s\n", message); exit(EXIT_FAILURE); } -static cJSON *read_json(const char *path) { +cJSON *read_json(const char *path) { FILE *file = fopen(path, "rb"); long size = 0; char *text; @@ -66,7 +80,7 @@ static cJSON *read_json(const char *path) { return json; } -static cJSON *input_header(const char *path) { +cJSON *input_header(const char *path) { hid_t file = H5Fopen(path, H5F_ACC_RDONLY, H5P_DEFAULT); char *text = NULL; cJSON *outer, *header; @@ -77,19 +91,68 @@ static cJSON *input_header(const char *path) { free(text); if (!cJSON_IsObject(outer)) die("invalid input Binsparse header"); header = cJSON_DetachItemFromObjectCaseSensitive(outer, "binsparse"); + if (header == NULL) { + header = outer; + outer = NULL; + } cJSON_Delete(outer); if (!cJSON_IsObject(header)) die("input has no binsparse header"); return header; } -static bool header_values_are_iso(cJSON *header) { +bool header_values_are_iso(cJSON *header) { cJSON *types = cJSON_GetObjectItemCaseSensitive(header, "data_types"); const char *value = cJSON_GetStringValue( cJSON_GetObjectItemCaseSensitive(types, "values")); return value != NULL && strncmp(value, "iso[", 4) == 0; } -static void push_entry(entries_t *entries, const size_t *coord, size_t value) { +bool header_has_fill(cJSON *header) { + return cJSON_IsTrue(cJSON_GetObjectItemCaseSensitive(header, "fill")); +} + +static bsp_error_t read_fill_value(const char *path, bsp_array_t *value) { + hid_t file = H5Fopen(path, H5F_ACC_RDONLY, H5P_DEFAULT); + bsp_error_t error; + if (file < 0) return BSP_ERROR_IO; + error = bsp_read_array(value, file, "fill_value"); + H5Fclose(file); + return error; +} + +/* Preserve every supplied partial-header field and add its fill-value buffer. */ +void complete_output(const char *path, cJSON *requested, + const bsp_array_t *fill_value) { + hid_t file = H5Fopen(path, H5F_ACC_RDWR, H5P_DEFAULT); + char *text = NULL, *updated; + cJSON *outer, *header, *item; + if (file < 0 || bsp_read_attribute(&text, file, "binsparse") != BSP_SUCCESS) + die("cannot reopen output header"); + outer = cJSON_Parse(text); + free(text); + header = cJSON_GetObjectItemCaseSensitive(outer, "binsparse"); + cJSON_ArrayForEach(item, requested) { + cJSON *copy = cJSON_Duplicate(item, 1); + if (cJSON_HasObjectItem(header, item->string)) + cJSON_ReplaceItemInObjectCaseSensitive(header, item->string, copy); + else + cJSON_AddItemToObject(header, item->string, copy); + } + if (fill_value != NULL && + bsp_write_array(file, "fill_value", *fill_value, 0) != BSP_SUCCESS) + die("cannot write fill_value"); + /* Compliance containers store the Binsparse header directly. */ + updated = cJSON_Print(header); + H5Adelete(file, "binsparse"); + if (updated == NULL || + bsp_write_attribute(file, "binsparse", updated) != BSP_SUCCESS) + die("cannot update output header"); + free(updated); + cJSON_Delete(outer); + H5Fclose(file); +} + +void push_entry(entries_t *entries, const size_t *coord, size_t value) { entry_t *item; if (entries->size == entries->capacity) { size_t capacity = entries->capacity == 0 ? 16 : 2 * entries->capacity; @@ -99,13 +162,14 @@ static void push_entry(entries_t *entries, const size_t *coord, size_t value) { entries->capacity = capacity; } item = &entries->entry[entries->size++]; - item->coord = malloc((size_t) entries->rank * sizeof(*item->coord)); + item->coord = malloc((entries->rank ? (size_t) entries->rank : 1) * + sizeof(*item->coord)); if (item->coord == NULL) die("out of memory"); memcpy(item->coord, coord, (size_t) entries->rank * sizeof(*coord)); item->value = value; } -static void free_entries(entries_t *entries) { +void free_entries(entries_t *entries) { for (size_t i = 0; i < entries->size; ++i) free(entries->entry[i].coord); free(entries->entry); } @@ -188,18 +252,18 @@ static void level_to_coo(const bsp_level_t *level, const bsp_tensor_t *tensor, die("unknown tensor level"); } -static entries_t tensor_to_coo(const bsp_tensor_t *tensor) { +entries_t tensor_to_coo(const bsp_tensor_t *tensor) { entries_t result = {0}; size_t *root; result.rank = tensor->rank; - root = calloc((size_t) tensor->rank, sizeof(*root)); + root = calloc(tensor->rank ? (size_t) tensor->rank : 1, sizeof(*root)); if (root == NULL) die("out of memory"); level_to_coo(tensor->level, tensor, 0, 1, root, &result); free(root); return result; } -static entries_t matrix_to_coo(bsp_matrix_t *matrix) { +entries_t matrix_to_coo(bsp_matrix_t *matrix) { entries_t result = {0}; result.rank = bsp_matrix_format_is_vector(matrix->format) ? 1 : 2; if (matrix->format == BSP_COOR || matrix->format == BSP_COOC || @@ -260,8 +324,8 @@ static entries_t matrix_to_coo(bsp_matrix_t *matrix) { } /* A predefined format is only a named view of a custom level tree. */ -static bsp_error_t write_predefined(const char *path, bsp_tensor_t tensor, - const char *format, int compression) { +bsp_error_t write_predefined(const char *path, bsp_tensor_t tensor, + const char *format, int compression) { bsp_matrix_t matrix; bsp_level_t *level = tensor.level; bsp_construct_default_matrix_t(&matrix); @@ -322,9 +386,10 @@ static int compare_entry(const void *a_, const void *b_) { return 0; } -static void apply_transpose(entries_t *entries, const size_t *source, - const size_t *target) { - size_t *copy = malloc((size_t) entries->rank * sizeof(*copy)); +void apply_transpose(entries_t *entries, const size_t *source, + const size_t *target) { + size_t *copy = + malloc((entries->rank ? (size_t) entries->rank : 1) * sizeof(*copy)); if (copy == NULL) die("out of memory"); for (size_t k = 0; k < entries->size; ++k) { memcpy(copy, entries->entry[k].coord, @@ -364,10 +429,10 @@ static size_t prefix_equal(const entry_t *a, const entry_t *b, size_t depth, /* Recursively build a level. parent_ptr partitions entries into incoming * fibers. The entries are already lexicographically sorted. */ -static bsp_level_t *build_level(cJSON *description, const entries_t *entries, - const size_t *dims, size_t depth, - const size_t *parent_ptr, size_t parents, - bool root, bsp_array_t values) { +bsp_level_t *build_level(const reformat_context_t *context, cJSON *description, + const entries_t *entries, const size_t *dims, + size_t depth, const size_t *parent_ptr, size_t parents, + bool root, bsp_array_t values) { cJSON *desc = cJSON_GetObjectItemCaseSensitive(description, "level_desc"); bsp_level_t *level = calloc(1, sizeof(*level)); const char *name = cJSON_GetStringValue(desc); @@ -404,16 +469,25 @@ static bsp_level_t *build_level(cJSON *description, const entries_t *entries, sparse->indices = calloc((size_t) rank, sizeof(*sparse->indices)); child_ptr = malloc((groups + 1) * sizeof(*child_ptr)); if (sparse->indices == NULL || child_ptr == NULL) die("out of memory"); - for (int d = 0; d < rank; ++d) - if (bsp_construct_array_t(&sparse->indices[d], groups, - bsp_pick_integer_type(dims[depth + (size_t) d])) != + for (int d = 0; d < rank; ++d) { + char key[64]; + snprintf(key, sizeof(key), "indices_%zu", depth + (size_t) d); + if (bsp_construct_array_t( + &sparse->indices[d], groups, + requested_type(context, key, + bsp_pick_integer_type(dims[depth + (size_t) d]))) != BSP_SUCCESS) die("out of memory"); + } if (!root) { + char key[64]; + snprintf(key, sizeof(key), "pointers_to_%zu", depth); sparse->pointers_to = malloc(sizeof(*sparse->pointers_to)); if (sparse->pointers_to == NULL || bsp_construct_array_t(sparse->pointers_to, parents + 1, - bsp_pick_integer_type(groups)) != BSP_SUCCESS) + requested_type(context, key, + bsp_pick_integer_type(groups))) != + BSP_SUCCESS) die("out of memory"); } groups = 0; @@ -440,8 +514,9 @@ static bsp_level_t *build_level(cJSON *description, const entries_t *entries, bsp_array_t pointers = *sparse->pointers_to; bsp_array_write(pointers, parents, groups); } - sparse->child = build_level(child_desc, entries, dims, depth + (size_t) rank, - child_ptr, groups, false, values); + sparse->child = build_level(context, child_desc, entries, dims, + depth + (size_t) rank, child_ptr, groups, false, + values); free(child_ptr); level->kind = BSP_TENSOR_SPARSE; level->data = sparse; @@ -477,7 +552,7 @@ static bsp_level_t *build_level(cJSON *description, const entries_t *entries, } child_ptr[children] = entries->size; dense->rank = rank; - dense->child = build_level(child_desc, entries, dims, + dense->child = build_level(context, child_desc, entries, dims, depth + (size_t) rank, child_ptr, children, false, values); free(child_ptr); @@ -489,7 +564,7 @@ static bsp_level_t *build_level(cJSON *description, const entries_t *entries, return NULL; } -static cJSON *predefined_level(const char *format, size_t *transpose, int rank) { +cJSON *predefined_level(const char *format, size_t *transpose, int rank) { const char *json = NULL; if (!strcmp(format, "DVEC")) json = "{\"level_desc\":\"dense\",\"rank\":1,\"level\":{\"level_desc\":\"element\"}}"; else if (!strcmp(format, "CVEC")) json = "{\"level_desc\":\"sparse\",\"rank\":1,\"level\":{\"level_desc\":\"element\"}}"; @@ -505,35 +580,39 @@ static cJSON *predefined_level(const char *format, size_t *transpose, int rank) return cJSON_Parse(json); } -static void usage(void) { - fprintf(stderr, "usage: reformat INPUT OUTPUT TARGET_HEADER.json [COMPRESSION]\n"); - exit(2); -} - -int main(int argc, char **argv) { - cJSON *source_header, *target_header, *format_json, *custom, *description; +bsp_error_t bsp_reformat_file(const char *input, const char *output, + cJSON *target_header, int compression) { + cJSON *source_header, *format_json, *custom, *description; + reformat_context_t context; const char *source_format, *target_format; bsp_tensor_t tensor = bsp_construct_default_tensor_t(); bsp_matrix_t matrix; bsp_array_t source_values, values; entries_t entries; size_t *source_transpose, *target_transpose, *stored_dims, root_ptr[2]; - int rank, compression = 0; + int rank; bool source_is_tensor, iso; - if (argc < 4 || argc > 5) usage(); - if (argc == 5) compression = atoi(argv[4]); - if (compression < 0 || compression > 9) die("compression must be 0 through 9"); - - source_header = input_header(argv[1]); - target_header = read_json(argv[3]); + bsp_array_t fill_value; + bool owns_fill = false; + if (compression < 0 || compression > 9) + return BSP_ERROR_INVALID_INPUT; + + source_header = input_header(input); + context.data_types = + cJSON_GetObjectItemCaseSensitive(target_header, "data_types"); format_json = cJSON_GetObjectItemCaseSensitive(source_header, "format"); source_format = cJSON_GetStringValue(format_json); target_format = cJSON_GetStringValue(cJSON_GetObjectItemCaseSensitive(target_header, "format")); if (source_format == NULL || target_format == NULL) die("header has no format string"); + if (header_has_fill(source_header)) { + if (read_fill_value(input, &fill_value) != BSP_SUCCESS) + die("cannot read input fill_value"); + owns_fill = true; + } source_is_tensor = strcmp(source_format, "custom") == 0; if (source_is_tensor) { - tensor = bsp_read_tensor(argv[1], NULL); + tensor = bsp_read_tensor(input, NULL); if (tensor.level == NULL) die("failed to parse input tensor"); tensor.is_iso = header_values_are_iso(source_header); rank = tensor.rank; @@ -541,7 +620,7 @@ int main(int argc, char **argv) { iso = tensor.is_iso; entries = tensor_to_coo(&tensor); } else { - if (bsp_read_matrix(&matrix, argv[1], NULL) != BSP_SUCCESS) + if (bsp_read_matrix(&matrix, input, NULL) != BSP_SUCCESS) die("failed to parse input matrix"); rank = bsp_matrix_format_is_vector(matrix.format) ? 1 : 2; source_values = matrix.values; @@ -549,9 +628,10 @@ int main(int argc, char **argv) { entries = matrix_to_coo(&matrix); } - source_transpose = malloc((size_t) rank * sizeof(*source_transpose)); - target_transpose = malloc((size_t) rank * sizeof(*target_transpose)); - stored_dims = malloc((size_t) rank * sizeof(*stored_dims)); + size_t allocated_rank = rank ? (size_t) rank : 1; + source_transpose = malloc(allocated_rank * sizeof(*source_transpose)); + target_transpose = malloc(allocated_rank * sizeof(*target_transpose)); + stored_dims = malloc(allocated_rank * sizeof(*stored_dims)); if (!source_transpose || !target_transpose || !stored_dims) die("out of memory"); for (int d = 0; d < rank; ++d) { source_transpose[d] = source_is_tensor && tensor.transpose ? tensor.transpose[d] : (size_t) d; @@ -592,29 +672,32 @@ int main(int argc, char **argv) { result.nnz = entries.size; result.is_iso = iso; result.structure = source_is_tensor ? tensor.structure : matrix.structure; - result.dims = malloc((size_t) rank * sizeof(*result.dims)); - result.transpose = malloc((size_t) rank * sizeof(*result.transpose)); + result.dims = malloc(allocated_rank * sizeof(*result.dims)); + result.transpose = malloc(allocated_rank * sizeof(*result.transpose)); if (!result.dims || !result.transpose) die("out of memory"); for (int d = 0; d < rank; ++d) { result.dims[d] = source_is_tensor ? tensor.dims[d] : (d == 0 ? matrix.nrows : matrix.ncols); result.transpose[d] = target_transpose[d]; } root_ptr[0] = 0; root_ptr[1] = entries.size; - result.level = build_level(description, &entries, stored_dims, 0, root_ptr, 1, true, values); + result.level = build_level(&context, description, &entries, stored_dims, 0, + root_ptr, 1, true, values); if (!strcmp(target_format, "custom")) { cJSON *user = cJSON_CreateObject(); - if (bsp_write_tensor(argv[2], result, NULL, user, compression) != BSP_SUCCESS) + if (bsp_write_tensor(output, result, NULL, user, compression) != BSP_SUCCESS) die("failed to write output tensor"); cJSON_Delete(user); - } else if (write_predefined(argv[2], result, target_format, compression) != + } else if (write_predefined(output, result, target_format, compression) != BSP_SUCCESS) { die("failed to write predefined output"); } + complete_output(output, target_header, owns_fill ? &fill_value : NULL); bsp_destroy_tensor_t(result); free_entries(&entries); if (source_is_tensor) bsp_destroy_tensor_t(tensor); else bsp_destroy_matrix_t(&matrix); + if (owns_fill) bsp_destroy_array_t(&fill_value); free(source_transpose); free(target_transpose); free(stored_dims); - cJSON_Delete(description); cJSON_Delete(source_header); cJSON_Delete(target_header); - return 0; + cJSON_Delete(description); cJSON_Delete(source_header); + return BSP_SUCCESS; } diff --git a/test/compliance/reformat.h b/test/compliance/reformat.h new file mode 100644 index 0000000..62f77e4 --- /dev/null +++ b/test/compliance/reformat.h @@ -0,0 +1,55 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +typedef struct { + size_t *coord; + size_t value; +} entry_t; + +typedef struct { + entry_t *entry; + size_t size; + size_t capacity; + int rank; +} entries_t; + +typedef struct { + cJSON *data_types; +} reformat_context_t; + +void die(const char *message); +cJSON *read_json(const char *path); +cJSON *input_header(const char *path); +bool header_values_are_iso(cJSON *header); +bool header_has_fill(cJSON *header); +void complete_output(const char *path, cJSON *requested, + const bsp_array_t *fill_value); + +bsp_type_t type_from_name(const char *name); +void push_entry(entries_t *entries, const size_t *coord, size_t value); +void free_entries(entries_t *entries); +entries_t tensor_to_coo(const bsp_tensor_t *tensor); +entries_t matrix_to_coo(bsp_matrix_t *matrix); +void apply_transpose(entries_t *entries, const size_t *source, + const size_t *target); + +cJSON *predefined_level(const char *format, size_t *transpose, int rank); +bsp_level_t *build_level(const reformat_context_t *context, + cJSON *description, const entries_t *entries, + const size_t *dims, size_t depth, + const size_t *parent_ptr, size_t parents, bool root, + bsp_array_t values); +bsp_error_t write_predefined(const char *path, bsp_tensor_t tensor, + const char *format, int compression); + +bsp_error_t bsp_reformat_file(const char *input, const char *output, + cJSON *target_header, int compression); diff --git a/test/compliance/run-binsparse-tests.sh b/test/compliance/run-binsparse-tests.sh new file mode 100755 index 0000000..9705ad7 --- /dev/null +++ b/test/compliance/run-binsparse-tests.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash + +# SPDX-FileCopyrightText: 2026 Binsparse Developers +# SPDX-License-Identifier: BSD-3-Clause + +set -euo pipefail + +script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +repo_root="$(cd -- "${script_dir}/../.." && pwd)" +build_dir="${BINSPARSE_BUILD_DIR:-${repo_root}/build-compliance}" +tests_dir="${BINSPARSE_TESTS_DIR:-${build_dir}/binsparse-tests}" +tests_ref="${BINSPARSE_TESTS_REF:-main}" +build_config="${BINSPARSE_BUILD_CONFIG:-Release}" + +for command in git pixi; do + if ! command -v "${command}" >/dev/null 2>&1; then + echo "error: required command '${command}' was not found" >&2 + exit 1 + fi +done + +if [[ -d "${tests_dir}/.git" ]]; then + git -C "${tests_dir}" fetch --depth 1 origin "${tests_ref}" + git -C "${tests_dir}" checkout --detach FETCH_HEAD +elif [[ -e "${tests_dir}" ]]; then + echo "error: ${tests_dir} exists but is not a Git checkout" >&2 + exit 1 +else + git clone --depth 1 --branch "${tests_ref}" \ + https://github.com/Binsparse/binsparse-tests.git "${tests_dir}" +fi + +( + cd "${tests_dir}" + pixi install -e test +) + +build_environment=(pixi exec -s cmake -s hdf5 -s zlib) +"${build_environment[@]}" cmake -S "${repo_root}" -B "${build_dir}" \ + -DCMAKE_BUILD_TYPE="${build_config}" +"${build_environment[@]}" cmake --build "${build_dir}" \ + --config "${build_config}" --parallel \ + --target npy_to_binsparse binsparse_to_npy binsparse_to_binsparse + +executable_dir="${build_dir}/test/compliance" +if [[ -d "${executable_dir}/${build_config}" ]]; then + executable_dir="${executable_dir}/${build_config}" +fi + +for executable in npy_to_binsparse binsparse_to_npy binsparse_to_binsparse; do + if [[ ! -x "${executable_dir}/${executable}" ]]; then + echo "error: missing executable ${executable_dir}/${executable}" >&2 + exit 1 + fi +done + +export BINSPARSE_BIN="${executable_dir}" +( + cd "${tests_dir}" + pixi run -e test pytest -m hdf5 binsparse_tests/ "$@" +) diff --git a/third_party/cnpy/cnpy.cpp b/third_party/cnpy/cnpy.cpp index 2d28578..902730c 100644 --- a/third_party/cnpy/cnpy.cpp +++ b/third_party/cnpy/cnpy.cpp @@ -25,6 +25,7 @@ char cnpy::map_type(const std::type_info& t) if(t == typeid(int) ) return 'i'; if(t == typeid(char) ) return 'i'; + if(t == typeid(signed char) ) return 'i'; if(t == typeid(short) ) return 'i'; if(t == typeid(long) ) return 'i'; if(t == typeid(long long) ) return 'i'; @@ -337,4 +338,3 @@ cnpy::NpyArray cnpy::npy_load(std::string fname) { } - diff --git a/third_party/cnpy/cnpy.h b/third_party/cnpy/cnpy.h index 0d3bb4c..08bd303 100644 --- a/third_party/cnpy/cnpy.h +++ b/third_party/cnpy/cnpy.h @@ -240,10 +240,12 @@ namespace cnpy { dict += map_type(typeid(T)); dict += std::to_string(sizeof(T)); dict += "', 'fortran_order': False, 'shape': ("; - dict += std::to_string(shape[0]); - for(size_t i = 1;i < shape.size();i++) { - dict += ", "; - dict += std::to_string(shape[i]); + if(!shape.empty()) { + dict += std::to_string(shape[0]); + for(size_t i = 1;i < shape.size();i++) { + dict += ", "; + dict += std::to_string(shape[i]); + } } if(shape.size() == 1) dict += ","; dict += "), }"; From e8a5335e27fcde1ee2af4bdbc3fa91dab26a4961 Mon Sep 17 00:00:00 2001 From: Willow Ahrens Date: Thu, 27 Aug 2026 13:55:40 -0400 Subject: [PATCH 04/16] seems good --- include/binsparse/detail/hdf5_types.h | 6 +++--- include/binsparse/hdf5_wrapper.h | 6 ++++-- src/read_matrix.c | 4 ++++ src/read_tensor.c | 18 +++++++++++++++++- test/compliance/binsparse_to_npy.c | 2 ++ test/compliance/reformat.c | 7 +------ 6 files changed, 31 insertions(+), 12 deletions(-) diff --git a/include/binsparse/detail/hdf5_types.h b/include/binsparse/detail/hdf5_types.h index 02f0315..0cf20c9 100644 --- a/include/binsparse/detail/hdf5_types.h +++ b/include/binsparse/detail/hdf5_types.h @@ -31,7 +31,7 @@ static inline hid_t bsp_get_hdf5_standard_type(bsp_type_t type) { } else if (type == BSP_FLOAT64) { return H5T_IEEE_F64LE; } else if (type == BSP_BINT8) { - return H5T_STD_I8LE; + return H5T_STD_U8LE; } else { return H5I_INVALID_HID; } @@ -86,7 +86,7 @@ static inline bsp_type_t bsp_get_bsp_type(hid_t type) { // way to generically determine the HDF5 native types for // stdint's fixed width integer types. static inline hid_t bsp_get_hdf5_native_type(bsp_type_t type) { - if (type == BSP_INT8 || type == BSP_BINT8) { + if (type == BSP_INT8) { if (sizeof(int8_t) == sizeof(char)) { return H5T_NATIVE_CHAR; } else if (sizeof(int8_t) == sizeof(short)) { @@ -142,7 +142,7 @@ static inline hid_t bsp_get_hdf5_native_type(bsp_type_t type) { } else { assert(false); } - } else if (type == BSP_UINT8) { + } else if (type == BSP_UINT8 || type == BSP_BINT8) { if (sizeof(uint8_t) == sizeof(unsigned char)) { return H5T_NATIVE_UCHAR; } else if (sizeof(uint8_t) == sizeof(unsigned short)) { diff --git a/include/binsparse/hdf5_wrapper.h b/include/binsparse/hdf5_wrapper.h index 4a2ce5d..aca19ed 100644 --- a/include/binsparse/hdf5_wrapper.h +++ b/include/binsparse/hdf5_wrapper.h @@ -237,8 +237,10 @@ static inline bsp_error_t bsp_read_array_allocator(bsp_array_t* array, hid_t f, return BSP_ERROR_MEMORY; } - herr_t status = H5Dread(dset, bsp_get_hdf5_native_type(type), H5S_ALL, - H5S_ALL, H5P_DEFAULT, array->data); + hid_t native_type = bsp_get_hdf5_native_type(type); + herr_t status = H5Dread(dset, native_type, H5S_ALL, H5S_ALL, H5P_DEFAULT, + array->data); + H5Tclose(hdf5_type); if (status < 0) { bsp_destroy_array_t(array); diff --git a/src/read_matrix.c b/src/read_matrix.c index 1fc3877..4671f28 100644 --- a/src/read_matrix.c +++ b/src/read_matrix.c @@ -148,6 +148,8 @@ bsp_error_t bsp_read_matrix_from_group_parallel(bsp_matrix_t* matrix, hid_t f, // TODO: handle error return error; } + } else if (strncmp(type_string, "bint8", 5) == 0) { + matrix->values.type = BSP_BINT8; } } @@ -308,6 +310,8 @@ bsp_error_t bsp_read_matrix_from_group_allocator(bsp_matrix_t* matrix, hid_t f, // TODO: handle error return error; } + } else if (strncmp(type_string, "bint8", 5) == 0) { + matrix->values.type = BSP_BINT8; } } diff --git a/src/read_tensor.c b/src/read_tensor.c index cce6004..939c94f 100644 --- a/src/read_tensor.c +++ b/src/read_tensor.c @@ -68,11 +68,17 @@ bsp_tensor_t bsp_read_tensor_from_group(hid_t f) { dims[idx] = cJSON_GetNumberValue(cJSON_GetArrayItem(shape_, idx)); } tensor.dims = dims; - assert(tensor.rank > 0); cJSON* data_types_ = cJSON_GetObjectItemCaseSensitive(binsparse, "data_types"); assert(data_types_ != NULL); + const char* values_type = cJSON_GetStringValue( + cJSON_GetObjectItemCaseSensitive(data_types_, "values")); + assert(values_type != NULL); + if (strncmp(values_type, "iso[", 4) == 0) { + tensor.is_iso = true; + values_type += 4; + } cJSON* binsparse_custom = cJSON_GetObjectItemCaseSensitive(binsparse, "custom"); @@ -113,6 +119,16 @@ bsp_tensor_t bsp_read_tensor_from_group(hid_t f) { free(json_string); return tensor; } + if (strncmp(values_type, "complex[", 8) == 0) { + error = bsp_fp_array_to_complex(&values); + if (error != BSP_SUCCESS) { + bsp_destroy_array_t(&values); + free(json_string); + return tensor; + } + } else if (strncmp(values_type, "bint8", 5) == 0) { + values.type = BSP_BINT8; + } cur_level->kind = BSP_TENSOR_ELEMENT; bsp_element_t* data = malloc(sizeof(bsp_element_t)); data->values = values; diff --git a/test/compliance/binsparse_to_npy.c b/test/compliance/binsparse_to_npy.c index 448fd55..0e44006 100644 --- a/test/compliance/binsparse_to_npy.c +++ b/test/compliance/binsparse_to_npy.c @@ -75,6 +75,8 @@ int main(int argc, char **argv) { if (fill_type != NULL && strncmp(fill_type, "complex[", 8) == 0 && bsp_fp_array_to_complex(&fill_value) != BSP_SUCCESS) die("invalid complex fill_value"); + if (fill_type != NULL && strncmp(fill_type, "bint8", 5) == 0) + fill_value.type = BSP_BINT8; owns_fill = true; } else { if (bsp_construct_array_t(&fill_value, 1, values.type) != BSP_SUCCESS) diff --git a/test/compliance/reformat.c b/test/compliance/reformat.c index bad1a30..852ab70 100644 --- a/test/compliance/reformat.c +++ b/test/compliance/reformat.c @@ -91,10 +91,6 @@ cJSON *input_header(const char *path) { free(text); if (!cJSON_IsObject(outer)) die("invalid input Binsparse header"); header = cJSON_DetachItemFromObjectCaseSensitive(outer, "binsparse"); - if (header == NULL) { - header = outer; - outer = NULL; - } cJSON_Delete(outer); if (!cJSON_IsObject(header)) die("input has no binsparse header"); return header; @@ -141,8 +137,7 @@ void complete_output(const char *path, cJSON *requested, if (fill_value != NULL && bsp_write_array(file, "fill_value", *fill_value, 0) != BSP_SUCCESS) die("cannot write fill_value"); - /* Compliance containers store the Binsparse header directly. */ - updated = cJSON_Print(header); + updated = cJSON_Print(outer); H5Adelete(file, "binsparse"); if (updated == NULL || bsp_write_attribute(file, "binsparse", updated) != BSP_SUCCESS) From 37154d73e6bc2c73eb54a520a47f2dfbbececaee Mon Sep 17 00:00:00 2001 From: Willow Ahrens Date: Thu, 27 Aug 2026 17:17:37 -0400 Subject: [PATCH 05/16] fix --- test/compliance/cnpy_c.cpp | 19 +++++++++++++++++-- test/compliance/cnpy_c.h | 2 +- test/compliance/npy_to_binsparse.c | 19 +++++++++++++++---- test/compliance/reformat.c | 18 ++++++++++++------ test/compliance/reformat.h | 2 ++ 5 files changed, 47 insertions(+), 13 deletions(-) diff --git a/test/compliance/cnpy_c.cpp b/test/compliance/cnpy_c.cpp index 8354aeb..835844c 100644 --- a/test/compliance/cnpy_c.cpp +++ b/test/compliance/cnpy_c.cpp @@ -14,11 +14,25 @@ extern "C" int cnpy_c_load(const char *path, cnpy_c_array *out) { out->size = array.num_vals; out->word_size = array.word_size; out->rank = array.shape.size(); - out->fortran_order = array.fortran_order; out->shape = static_cast(malloc(out->rank * sizeof(size_t))); + out->strides = static_cast(malloc(out->rank * sizeof(size_t))); out->data = malloc(array.num_bytes()); - if ((out->rank && !out->shape) || (array.num_bytes() && !out->data)) return 1; + if ((out->rank && (!out->shape || !out->strides)) || + (array.num_bytes() && !out->data)) + return 1; if (out->rank) memcpy(out->shape, array.shape.data(), out->rank * sizeof(size_t)); + size_t stride = 1; + if (array.fortran_order) { + for (size_t d = 0; d < out->rank; ++d) { + out->strides[d] = stride; + stride *= out->shape[d]; + } + } else { + for (size_t d = out->rank; d-- > 0;) { + out->strides[d] = stride; + stride *= out->shape[d]; + } + } if (array.num_bytes()) memcpy(out->data, array.data(), array.num_bytes()); return 0; } catch (const std::exception &) { @@ -29,6 +43,7 @@ extern "C" int cnpy_c_load(const char *path, cnpy_c_array *out) { extern "C" void cnpy_c_destroy(cnpy_c_array *array) { free(array->data); free(array->shape); + free(array->strides); memset(array, 0, sizeof(*array)); } diff --git a/test/compliance/cnpy_c.h b/test/compliance/cnpy_c.h index bb1b2d7..e7921e5 100644 --- a/test/compliance/cnpy_c.h +++ b/test/compliance/cnpy_c.h @@ -12,7 +12,7 @@ typedef struct { size_t word_size; size_t rank; size_t *shape; - int fortran_order; + size_t *strides; } cnpy_c_array; int cnpy_c_load(const char *path, cnpy_c_array *array); diff --git a/test/compliance/npy_to_binsparse.c b/test/compliance/npy_to_binsparse.c index 0b858cc..d3d7f1d 100644 --- a/test/compliance/npy_to_binsparse.c +++ b/test/compliance/npy_to_binsparse.c @@ -12,6 +12,13 @@ static size_t product(const size_t *shape, size_t rank) { return size; } +static size_t offset(const cnpy_c_array *array, const size_t *coord) { + size_t result = 0; + for (size_t d = 0; d < array->rank; ++d) + result += coord[d] * array->strides[d]; + return result; +} + int main(int argc, char **argv) { cnpy_c_array dense = {0}, pattern = {0}, fill = {0}; cJSON *header, *shape_json, *custom, *description; @@ -34,8 +41,6 @@ int main(int argc, char **argv) { if (cnpy_c_load(argv[1], &dense) || cnpy_c_load(argv[2], &pattern) || cnpy_c_load(argv[3], &fill)) die("cannot read NPY input"); - if (dense.fortran_order || pattern.fortran_order || fill.fortran_order) - die("Fortran-order NPY arrays are unsupported"); if (dense.rank != pattern.rank || dense.size != pattern.size || pattern.word_size != 1 || fill.size != 1) die("incompatible tensor, pattern, or fill-value NPY input"); @@ -81,11 +86,14 @@ int main(int argc, char **argv) { size_t value_index = 0; for (size_t linear = 0; linear < dense.size; ++linear) { - if (((uint8_t *) pattern.data)[linear]) { + size_t dense_offset = offset(&dense, logical_coord); + size_t pattern_offset = offset(&pattern, logical_coord); + if (((uint8_t *) pattern.data)[pattern_offset]) { push_entry(&entries, logical_coord, iso ? 0 : value_index); if (!iso || value_index == 0) memcpy((char *) values.data + (iso ? 0 : value_index) * dense.word_size, - (char *) dense.data + linear * dense.word_size, dense.word_size); + (char *) dense.data + dense_offset * dense.word_size, + dense.word_size); ++value_index; } for (int d = result.rank - 1; d >= 0; --d) { @@ -114,6 +122,9 @@ int main(int argc, char **argv) { for (int d = 0; d < result.rank; ++d) result.transpose[d] = target_transpose[d]; } apply_transpose(&entries, source_transpose, target_transpose); + bsp_array_t ordered_values = values_in_entry_order(values, &entries, iso); + bsp_destroy_array_t(&values); + values = ordered_values; for (int d = 0; d < result.rank; ++d) stored_dims[d] = result.dims[target_transpose[d]]; root_ptr[0] = 0; root_ptr[1] = stored; result.level = build_level(&context, description, &entries, stored_dims, 0, diff --git a/test/compliance/reformat.c b/test/compliance/reformat.c index 852ab70..26c08e4 100644 --- a/test/compliance/reformat.c +++ b/test/compliance/reformat.c @@ -171,6 +171,12 @@ void free_entries(entries_t *entries) { /* Expand one custom level. begin/end identify the parent positions entering * the level; coordinates are stored row-wise in coords. */ +static size_t stored_dimension(const bsp_tensor_t *tensor, size_t depth) { + size_t logical = tensor->transpose ? tensor->transpose[depth] : depth; + if (logical >= (size_t) tensor->rank) die("transpose is not a permutation"); + return tensor->dims[logical]; +} + static void level_to_coo(const bsp_level_t *level, const bsp_tensor_t *tensor, size_t depth, size_t parents, const size_t *coords, entries_t *out) { @@ -189,7 +195,7 @@ static void level_to_coo(const bsp_level_t *level, const bsp_tensor_t *tensor, const bsp_dense_t *dense = level->data; size_t width = 1, count, *next; for (int d = 0; d < dense->rank; ++d) - width *= tensor->dims[depth + (size_t) d]; + width *= stored_dimension(tensor, depth + (size_t) d); count = parents * width; next = calloc(count * (size_t) tensor->rank, sizeof(*next)); if (next == NULL && count != 0) die("out of memory"); @@ -200,7 +206,7 @@ static void level_to_coo(const bsp_level_t *level, const bsp_tensor_t *tensor, coords + p * (size_t) tensor->rank, depth * sizeof(*next)); for (int d = dense->rank - 1; d >= 0; --d) { - size_t dim = tensor->dims[depth + (size_t) d]; + size_t dim = stored_dimension(tensor, depth + (size_t) d); next[q * (size_t) tensor->rank + depth + (size_t) d] = n % dim; n /= dim; } @@ -232,7 +238,7 @@ static void level_to_coo(const bsp_level_t *level, const bsp_tensor_t *tensor, for (int d = 0; d < sparse->rank; ++d) { size_t index = 0; bsp_array_read(sparse->indices[d], i, index); - if (index >= tensor->dims[depth + (size_t) d]) + if (index >= stored_dimension(tensor, depth + (size_t) d)) die("input coordinate is out of bounds"); next[q * (size_t) tensor->rank + depth + (size_t) d] = index; } @@ -401,8 +407,8 @@ void apply_transpose(entries_t *entries, const size_t *source, qsort(entries->entry, entries->size, sizeof(*entries->entry), compare_entry); } -static bsp_array_t copy_values(const bsp_array_t source, const entries_t *entries, - bool iso) { +bsp_array_t values_in_entry_order(const bsp_array_t source, + const entries_t *entries, bool iso) { bsp_array_t values; size_t size = iso && entries->size != 0 ? 1 : entries->size; if (bsp_construct_array_t(&values, size, source.type) != BSP_SUCCESS) @@ -661,7 +667,7 @@ bsp_error_t bsp_reformat_file(const char *input, const char *output, stored_dims[d] = source_is_tensor ? tensor.dims[logical] : (logical == 0 ? matrix.nrows : matrix.ncols); } - values = copy_values(source_values, &entries, iso); + values = values_in_entry_order(source_values, &entries, iso); bsp_tensor_t result = bsp_construct_default_tensor_t(); result.rank = rank; result.nnz = entries.size; diff --git a/test/compliance/reformat.h b/test/compliance/reformat.h index 62f77e4..ac2a10e 100644 --- a/test/compliance/reformat.h +++ b/test/compliance/reformat.h @@ -41,6 +41,8 @@ entries_t tensor_to_coo(const bsp_tensor_t *tensor); entries_t matrix_to_coo(bsp_matrix_t *matrix); void apply_transpose(entries_t *entries, const size_t *source, const size_t *target); +bsp_array_t values_in_entry_order(const bsp_array_t source, + const entries_t *entries, bool iso); cJSON *predefined_level(const char *format, size_t *transpose, int rank); bsp_level_t *build_level(const reformat_context_t *context, From 5c9f05d90c05cfb03df6a0612af5bd419d46431a Mon Sep 17 00:00:00 2001 From: Willow Ahrens Date: Thu, 27 Aug 2026 18:03:58 -0400 Subject: [PATCH 06/16] fix --- .github/workflows/ci.yml | 55 ++++++++++++++++++++------ test/compliance/run-binsparse-tests.sh | 7 ---- 2 files changed, 44 insertions(+), 18 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 516614f..f231a05 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,11 +29,13 @@ jobs: strategy: matrix: include: - - cc: gcc-12 + - id: gcc-12 + cc: gcc-12 cxx: g++-12 - - cc: clang + - id: clang + cc: clang cxx: clang++ - name: ${{ matrix.cc }} + name: Build (${{ matrix.cc }}) env: CC: ${{ matrix.cc }} CXX: ${{ matrix.cxx }} @@ -46,13 +48,44 @@ jobs: cmake -B build - name: Build run: VERBOSE=true make -C build -j `nproc` + - name: Archive Build + run: tar -czf build.tar.gz build + - uses: actions/upload-artifact@v4 + with: + name: compliance-build-${{ matrix.id }} + path: build.tar.gz + retention-days: 1 + + test: + name: Test (${{ matrix.cc }}) + needs: build + runs-on: ubuntu-latest + timeout-minutes: 60 + strategy: + matrix: + include: + - id: gcc-12 + cc: gcc-12 + - id: clang + cc: clang + steps: + - uses: actions/checkout@v4 + - uses: actions/download-artifact@v4 + with: + name: compliance-build-${{ matrix.id }} + - name: Restore Build + run: tar -xzf build.tar.gz + - name: Install Runtime Dependencies + run: | + sudo apt-get update + sudo apt-get install libhdf5-dev + - uses: prefix-dev/setup-pixi@v0.10.0 + with: + run-install: false + global-cache: true - name: Matrix Tests run: ctest --test-dir ./build/test/bash - - name: Install Julia - run: curl -fsSL https://install.julialang.org | sh -s -- -y - - name: Disable Julia precompilation - run: julia -e 'using Pkg; Pkg.add(["PrecompileTools", "Preferences"]); using PrecompileTools, Preferences; set_preferences!(PrecompileTools, "precompile_workloads" => false; force=true)' - - name: Install Julia Packages - run: julia -e 'using Pkg; Pkg.add(["Finch", "HDF5"])' - - name: Tensor Tests - run: ctest --test-dir ./build/test/julia + - name: Compliance Tests + env: + BINSPARSE_BUILD_DIR: ${{ github.workspace }}/build + run: ./test/compliance/run-binsparse-tests.sh -q diff --git a/test/compliance/run-binsparse-tests.sh b/test/compliance/run-binsparse-tests.sh index 9705ad7..c562653 100755 --- a/test/compliance/run-binsparse-tests.sh +++ b/test/compliance/run-binsparse-tests.sh @@ -35,13 +35,6 @@ fi pixi install -e test ) -build_environment=(pixi exec -s cmake -s hdf5 -s zlib) -"${build_environment[@]}" cmake -S "${repo_root}" -B "${build_dir}" \ - -DCMAKE_BUILD_TYPE="${build_config}" -"${build_environment[@]}" cmake --build "${build_dir}" \ - --config "${build_config}" --parallel \ - --target npy_to_binsparse binsparse_to_npy binsparse_to_binsparse - executable_dir="${build_dir}/test/compliance" if [[ -d "${executable_dir}/${build_config}" ]]; then executable_dir="${executable_dir}/${build_config}" From 815ad6ffebd0cba561b4bc8bf15a4dfffc1dcb83 Mon Sep 17 00:00:00 2001 From: Willow Ahrens Date: Fri, 28 Aug 2026 00:40:49 -0400 Subject: [PATCH 07/16] format --- .github/workflows/ci.yml | 2 +- LICENSES/MIT.txt | 21 + include/binsparse/hdf5_wrapper.h | 4 +- test/compliance/CMakeLists.txt | 6 + test/compliance/binsparse_to_binsparse.c | 10 +- test/compliance/binsparse_to_npy.c | 78 ++- test/compliance/cnpy_c.cpp | 88 +++- test/compliance/cnpy_c.h | 18 +- test/compliance/npy_to_binsparse.c | 103 ++-- test/compliance/reformat.c | 407 +++++++++------ test/compliance/reformat.h | 67 +-- third_party/cnpy/CMakeLists.txt | 6 + third_party/cnpy/README.md | 22 +- third_party/cnpy/cnpy.cpp | 626 ++++++++++++----------- third_party/cnpy/cnpy.h | 541 +++++++++++--------- third_party/cnpy/example1.cpp | 107 ++-- third_party/cnpy/mat2npz | 4 + third_party/cnpy/npy2mat | 4 + third_party/cnpy/npz2mat | 4 + 19 files changed, 1232 insertions(+), 886 deletions(-) create mode 100644 LICENSES/MIT.txt diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f231a05..98cf896 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -82,7 +82,7 @@ jobs: - uses: prefix-dev/setup-pixi@v0.10.0 with: run-install: false - global-cache: true + cache: false - name: Matrix Tests run: ctest --test-dir ./build/test/bash - name: Compliance Tests diff --git a/LICENSES/MIT.txt b/LICENSES/MIT.txt new file mode 100644 index 0000000..aa29325 --- /dev/null +++ b/LICENSES/MIT.txt @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2011 Carl Rogers + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/include/binsparse/hdf5_wrapper.h b/include/binsparse/hdf5_wrapper.h index aca19ed..b52423c 100644 --- a/include/binsparse/hdf5_wrapper.h +++ b/include/binsparse/hdf5_wrapper.h @@ -238,8 +238,8 @@ static inline bsp_error_t bsp_read_array_allocator(bsp_array_t* array, hid_t f, } hid_t native_type = bsp_get_hdf5_native_type(type); - herr_t status = H5Dread(dset, native_type, H5S_ALL, H5S_ALL, H5P_DEFAULT, - array->data); + herr_t status = + H5Dread(dset, native_type, H5S_ALL, H5S_ALL, H5P_DEFAULT, array->data); H5Tclose(hdf5_type); if (status < 0) { diff --git a/test/compliance/CMakeLists.txt b/test/compliance/CMakeLists.txt index 0ac3c0e..a3a639c 100644 --- a/test/compliance/CMakeLists.txt +++ b/test/compliance/CMakeLists.txt @@ -1,3 +1,9 @@ +#[[ +SPDX-FileCopyrightText: 2026 Binsparse Developers + +SPDX-License-Identifier: BSD-3-Clause +]] + find_package(ZLIB REQUIRED) add_library(cnpy_vendor STATIC diff --git a/test/compliance/binsparse_to_binsparse.c b/test/compliance/binsparse_to_binsparse.c index df71dea..e833fc9 100644 --- a/test/compliance/binsparse_to_binsparse.c +++ b/test/compliance/binsparse_to_binsparse.c @@ -1,9 +1,15 @@ +/* + * SPDX-FileCopyrightText: 2026 Binsparse Developers + * + * SPDX-License-Identifier: BSD-3-Clause + */ + #include "reformat.h" #include -int main(int argc, char **argv) { - cJSON *header; +int main(int argc, char** argv) { + cJSON* header; bsp_error_t error; if (argc != 3) { diff --git a/test/compliance/binsparse_to_npy.c b/test/compliance/binsparse_to_npy.c index 0e44006..d7af64c 100644 --- a/test/compliance/binsparse_to_npy.c +++ b/test/compliance/binsparse_to_npy.c @@ -1,5 +1,11 @@ -#include "reformat.h" +/* + * SPDX-FileCopyrightText: 2026 Binsparse Developers + * + * SPDX-License-Identifier: BSD-3-Clause + */ + #include "cnpy_c.h" +#include "reformat.h" #include #include @@ -7,23 +13,24 @@ #include #include -static size_t shape_size(const size_t *shape, int rank) { +static size_t shape_size(const size_t* shape, int rank) { size_t size = 1; - for (int d = 0; d < rank; ++d) size *= shape[d]; + for (int d = 0; d < rank; ++d) + size *= shape[d]; return size; } -int main(int argc, char **argv) { - cJSON *header; - const char *format; +int main(int argc, char** argv) { + cJSON* header; + const char* format; bool custom; bsp_tensor_t tensor = bsp_construct_default_tensor_t(); bsp_matrix_t matrix; bsp_array_t values, fill_value; entries_t entries; size_t *shape, *transpose, size; - void *dense; - uint8_t *pattern; + void* dense; + uint8_t* pattern; bool owns_fill = false; if (argc != 5) { @@ -32,8 +39,10 @@ int main(int argc, char **argv) { return 2; } header = input_header(argv[1]); - format = cJSON_GetStringValue(cJSON_GetObjectItemCaseSensitive(header, "format")); - if (format == NULL) die("input header has no format"); + format = + cJSON_GetStringValue(cJSON_GetObjectItemCaseSensitive(header, "format")); + if (format == NULL) + die("input header has no format"); custom = !strcmp(format, "custom"); int rank; if (custom) { @@ -49,29 +58,35 @@ int main(int argc, char **argv) { rank = bsp_matrix_format_is_vector(matrix.format) ? 1 : 2; shape = malloc((size_t) rank * sizeof(size_t)); shape[0] = matrix.nrows; - if (rank == 2) shape[1] = matrix.ncols; + if (rank == 2) + shape[1] = matrix.ncols; values = matrix.values; entries = matrix_to_coo(&matrix); } transpose = rank ? malloc((size_t) rank * sizeof(size_t)) : NULL; - for (int d = 0; d < rank; ++d) transpose[d] = (size_t) d; + for (int d = 0; d < rank; ++d) + transpose[d] = (size_t) d; if (custom && tensor.transpose) memcpy(transpose, tensor.transpose, (size_t) rank * sizeof(size_t)); else if (!custom && rank == 2 && (matrix.format == BSP_DMATC || matrix.format == BSP_CSC || matrix.format == BSP_DCSC || matrix.format == BSP_COOC)) { - transpose[0] = 1; transpose[1] = 0; + transpose[0] = 1; + transpose[1] = 0; } bsp_construct_default_array_t(&fill_value); - cJSON *fill = cJSON_GetObjectItemCaseSensitive(header, "fill"); + cJSON* fill = cJSON_GetObjectItemCaseSensitive(header, "fill"); if (cJSON_IsTrue(fill)) { hid_t file = H5Fopen(argv[1], H5F_ACC_RDONLY, H5P_DEFAULT); - if (file < 0 || bsp_read_array(&fill_value, file, "fill_value") != BSP_SUCCESS) + if (file < 0 || + bsp_read_array(&fill_value, file, "fill_value") != BSP_SUCCESS) die("cannot read fill_value"); H5Fclose(file); - const char *fill_type = cJSON_GetStringValue(cJSON_GetObjectItemCaseSensitive( - cJSON_GetObjectItemCaseSensitive(header, "data_types"), "fill_value")); + const char* fill_type = + cJSON_GetStringValue(cJSON_GetObjectItemCaseSensitive( + cJSON_GetObjectItemCaseSensitive(header, "data_types"), + "fill_value")); if (fill_type != NULL && strncmp(fill_type, "complex[", 8) == 0 && bsp_fp_array_to_complex(&fill_value) != BSP_SUCCESS) die("invalid complex fill_value"); @@ -87,20 +102,23 @@ int main(int argc, char **argv) { size = shape_size(shape, rank); dense = malloc(size * bsp_type_size(values.type)); pattern = calloc(size, 1); - if ((size && !dense) || (size && !pattern)) die("out of memory"); + if ((size && !dense) || (size && !pattern)) + die("out of memory"); for (size_t i = 0; i < size; ++i) - memcpy((char *) dense + i * bsp_type_size(values.type), fill_value.data, + memcpy((char*) dense + i * bsp_type_size(values.type), fill_value.data, bsp_type_size(values.type)); for (size_t k = 0; k < entries.size; ++k) { size_t linear = 0; for (int logical = 0; logical < rank; ++logical) { int stored = 0; - while (stored < rank && transpose[stored] != (size_t) logical) ++stored; + while (stored < rank && transpose[stored] != (size_t) logical) + ++stored; linear = linear * shape[logical] + entries.entry[k].coord[stored]; } pattern[linear] = 1; - memcpy((char *) dense + linear * bsp_type_size(values.type), - (char *) values.data + entries.entry[k].value * bsp_type_size(values.type), + memcpy((char*) dense + linear * bsp_type_size(values.type), + (char*) values.data + + entries.entry[k].value * bsp_type_size(values.type), bsp_type_size(values.type)); } if (cnpy_c_save(argv[2], dense, shape, (size_t) rank, values.type) || @@ -108,10 +126,18 @@ int main(int argc, char **argv) { cnpy_c_save(argv[4], fill_value.data, NULL, 0, fill_value.type)) die("cannot write NPY output"); - free(dense); free(pattern); free(transpose); free_entries(&entries); - if (owns_fill) bsp_destroy_array_t(&fill_value); - if (custom) bsp_destroy_tensor_t(tensor); - else { free(shape); bsp_destroy_matrix_t(&matrix); } + free(dense); + free(pattern); + free(transpose); + free_entries(&entries); + if (owns_fill) + bsp_destroy_array_t(&fill_value); + if (custom) + bsp_destroy_tensor_t(tensor); + else { + free(shape); + bsp_destroy_matrix_t(&matrix); + } cJSON_Delete(header); return 0; } diff --git a/test/compliance/cnpy_c.cpp b/test/compliance/cnpy_c.cpp index 835844c..273fc04 100644 --- a/test/compliance/cnpy_c.cpp +++ b/test/compliance/cnpy_c.cpp @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: 2026 Binsparse Developers +// +// SPDX-License-Identifier: BSD-3-Clause + #include "cnpy_c.h" #include @@ -8,19 +12,20 @@ #include #include -extern "C" int cnpy_c_load(const char *path, cnpy_c_array *out) { +extern "C" int cnpy_c_load(const char* path, cnpy_c_array* out) { try { cnpy::NpyArray array = cnpy::npy_load(path); out->size = array.num_vals; out->word_size = array.word_size; out->rank = array.shape.size(); - out->shape = static_cast(malloc(out->rank * sizeof(size_t))); - out->strides = static_cast(malloc(out->rank * sizeof(size_t))); + out->shape = static_cast(malloc(out->rank * sizeof(size_t))); + out->strides = static_cast(malloc(out->rank * sizeof(size_t))); out->data = malloc(array.num_bytes()); if ((out->rank && (!out->shape || !out->strides)) || (array.num_bytes() && !out->data)) return 1; - if (out->rank) memcpy(out->shape, array.shape.data(), out->rank * sizeof(size_t)); + if (out->rank) + memcpy(out->shape, array.shape.data(), out->rank * sizeof(size_t)); size_t stride = 1; if (array.fortran_order) { for (size_t d = 0; d < out->rank; ++d) { @@ -33,14 +38,15 @@ extern "C" int cnpy_c_load(const char *path, cnpy_c_array *out) { stride *= out->shape[d]; } } - if (array.num_bytes()) memcpy(out->data, array.data(), array.num_bytes()); + if (array.num_bytes()) + memcpy(out->data, array.data(), array.num_bytes()); return 0; - } catch (const std::exception &) { + } catch (const std::exception&) { return 1; } } -extern "C" void cnpy_c_destroy(cnpy_c_array *array) { +extern "C" void cnpy_c_destroy(cnpy_c_array* array) { free(array->data); free(array->shape); free(array->strides); @@ -48,34 +54,62 @@ extern "C" void cnpy_c_destroy(cnpy_c_array *array) { } template -static void save(const char *path, const void *data, const size_t *shape, +static void save(const char* path, const void* data, const size_t* shape, size_t rank) { std::vector dimensions; - if (rank != 0) dimensions.assign(shape, shape + rank); - cnpy::npy_save(path, static_cast(data), dimensions); + if (rank != 0) + dimensions.assign(shape, shape + rank); + cnpy::npy_save(path, static_cast(data), dimensions); } -extern "C" int cnpy_c_save(const char *path, const void *data, - const size_t *shape, size_t rank, int type) { +extern "C" int cnpy_c_save(const char* path, const void* data, + const size_t* shape, size_t rank, int type) { try { switch (type) { - case 0: save(path, data, shape, rank); break; - case 1: save(path, data, shape, rank); break; - case 2: save(path, data, shape, rank); break; - case 3: save(path, data, shape, rank); break; - case 4: save(path, data, shape, rank); break; - case 5: save(path, data, shape, rank); break; - case 6: save(path, data, shape, rank); break; - case 7: save(path, data, shape, rank); break; - case 8: save(path, data, shape, rank); break; - case 9: save(path, data, shape, rank); break; - case 10: save(path, data, shape, rank); break; - case 11: save>(path, data, shape, rank); break; - case 12: save>(path, data, shape, rank); break; - default: return 1; + case 0: + save(path, data, shape, rank); + break; + case 1: + save(path, data, shape, rank); + break; + case 2: + save(path, data, shape, rank); + break; + case 3: + save(path, data, shape, rank); + break; + case 4: + save(path, data, shape, rank); + break; + case 5: + save(path, data, shape, rank); + break; + case 6: + save(path, data, shape, rank); + break; + case 7: + save(path, data, shape, rank); + break; + case 8: + save(path, data, shape, rank); + break; + case 9: + save(path, data, shape, rank); + break; + case 10: + save(path, data, shape, rank); + break; + case 11: + save>(path, data, shape, rank); + break; + case 12: + save>(path, data, shape, rank); + break; + default: + return 1; } return 0; - } catch (const std::exception &) { + } catch (const std::exception&) { return 1; } } diff --git a/test/compliance/cnpy_c.h b/test/compliance/cnpy_c.h index e7921e5..13af018 100644 --- a/test/compliance/cnpy_c.h +++ b/test/compliance/cnpy_c.h @@ -1,3 +1,9 @@ +/* + * SPDX-FileCopyrightText: 2026 Binsparse Developers + * + * SPDX-License-Identifier: BSD-3-Clause + */ + #pragma once #include @@ -7,17 +13,17 @@ extern "C" { #endif typedef struct { - void *data; + void* data; size_t size; size_t word_size; size_t rank; - size_t *shape; - size_t *strides; + size_t* shape; + size_t* strides; } cnpy_c_array; -int cnpy_c_load(const char *path, cnpy_c_array *array); -void cnpy_c_destroy(cnpy_c_array *array); -int cnpy_c_save(const char *path, const void *data, const size_t *shape, +int cnpy_c_load(const char* path, cnpy_c_array* array); +void cnpy_c_destroy(cnpy_c_array* array); +int cnpy_c_save(const char* path, const void* data, const size_t* shape, size_t rank, int bsp_type); #ifdef __cplusplus diff --git a/test/compliance/npy_to_binsparse.c b/test/compliance/npy_to_binsparse.c index d3d7f1d..9ec16ca 100644 --- a/test/compliance/npy_to_binsparse.c +++ b/test/compliance/npy_to_binsparse.c @@ -1,25 +1,32 @@ -#include "reformat.h" +/* + * SPDX-FileCopyrightText: 2026 Binsparse Developers + * + * SPDX-License-Identifier: BSD-3-Clause + */ + #include "cnpy_c.h" +#include "reformat.h" #include #include #include #include -static size_t product(const size_t *shape, size_t rank) { +static size_t product(const size_t* shape, size_t rank) { size_t size = 1; - for (size_t d = 0; d < rank; ++d) size *= shape[d]; + for (size_t d = 0; d < rank; ++d) + size *= shape[d]; return size; } -static size_t offset(const cnpy_c_array *array, const size_t *coord) { +static size_t offset(const cnpy_c_array* array, const size_t* coord) { size_t result = 0; for (size_t d = 0; d < array->rank; ++d) result += coord[d] * array->strides[d]; return result; } -int main(int argc, char **argv) { +int main(int argc, char** argv) { cnpy_c_array dense = {0}, pattern = {0}, fill = {0}; cJSON *header, *shape_json, *custom, *description; const char *format, *value_name; @@ -48,18 +55,28 @@ int main(int argc, char **argv) { header = read_json(argv[4]); shape_json = cJSON_GetObjectItemCaseSensitive(header, "shape"); result.rank = cJSON_GetArraySize(shape_json); - if ((size_t) result.rank != dense.rank) die("header shape rank mismatch"); - result.dims = result.rank ? malloc((size_t) result.rank * sizeof(size_t)) : NULL; - result.transpose = result.rank ? malloc((size_t) result.rank * sizeof(size_t)) : NULL; - logical_coord = result.rank ? calloc((size_t) result.rank, sizeof(size_t)) : NULL; - target_transpose = result.rank ? malloc((size_t) result.rank * sizeof(size_t)) : NULL; - source_transpose = result.rank ? malloc((size_t) result.rank * sizeof(size_t)) : NULL; - stored_dims = result.rank ? malloc((size_t) result.rank * sizeof(size_t)) : NULL; + if ((size_t) result.rank != dense.rank) + die("header shape rank mismatch"); + result.dims = + result.rank ? malloc((size_t) result.rank * sizeof(size_t)) : NULL; + result.transpose = + result.rank ? malloc((size_t) result.rank * sizeof(size_t)) : NULL; + logical_coord = + result.rank ? calloc((size_t) result.rank, sizeof(size_t)) : NULL; + target_transpose = + result.rank ? malloc((size_t) result.rank * sizeof(size_t)) : NULL; + source_transpose = + result.rank ? malloc((size_t) result.rank * sizeof(size_t)) : NULL; + stored_dims = + result.rank ? malloc((size_t) result.rank * sizeof(size_t)) : NULL; entries.rank = result.rank; for (int d = 0; d < result.rank; ++d) { - result.dims[d] = (size_t) cJSON_GetNumberValue(cJSON_GetArrayItem(shape_json, d)); - if (result.dims[d] != dense.shape[d]) die("header shape mismatch"); - source_transpose[d] = result.transpose[d] = target_transpose[d] = (size_t) d; + result.dims[d] = + (size_t) cJSON_GetNumberValue(cJSON_GetArrayItem(shape_json, d)); + if (result.dims[d] != dense.shape[d]) + die("header shape mismatch"); + source_transpose[d] = result.transpose[d] = target_transpose[d] = + (size_t) d; } if (product(result.dims, (size_t) result.rank) != dense.size) die("NPY element count does not match shape"); @@ -68,7 +85,8 @@ int main(int argc, char **argv) { value_name = cJSON_GetStringValue( cJSON_GetObjectItemCaseSensitive(context.data_types, "values")); value_type = type_from_name(value_name); - if (value_type == BSP_INVALID_TYPE || bsp_type_size(value_type) != dense.word_size) + if (value_type == BSP_INVALID_TYPE || + bsp_type_size(value_type) != dense.word_size) die("NPY dtype does not match header values type"); if (fill.word_size != bsp_type_size(value_type)) die("fill-value dtype does not match header values type"); @@ -79,7 +97,8 @@ int main(int argc, char **argv) { bsp_get_type_string(value_type)); iso = value_name != NULL && !strncmp(value_name, "iso[", 4); for (size_t i = 0; i < pattern.size; ++i) - if (((uint8_t *) pattern.data)[i]) ++stored; + if (((uint8_t*) pattern.data)[i]) + ++stored; if (bsp_construct_array_t(&values, iso && stored ? 1 : stored, value_type) != BSP_SUCCESS) die("out of memory"); @@ -88,49 +107,57 @@ int main(int argc, char **argv) { for (size_t linear = 0; linear < dense.size; ++linear) { size_t dense_offset = offset(&dense, logical_coord); size_t pattern_offset = offset(&pattern, logical_coord); - if (((uint8_t *) pattern.data)[pattern_offset]) { + if (((uint8_t*) pattern.data)[pattern_offset]) { push_entry(&entries, logical_coord, iso ? 0 : value_index); if (!iso || value_index == 0) - memcpy((char *) values.data + (iso ? 0 : value_index) * dense.word_size, - (char *) dense.data + dense_offset * dense.word_size, + memcpy((char*) values.data + (iso ? 0 : value_index) * dense.word_size, + (char*) dense.data + dense_offset * dense.word_size, dense.word_size); ++value_index; } for (int d = result.rank - 1; d >= 0; --d) { - if (++logical_coord[d] < result.dims[d]) break; + if (++logical_coord[d] < result.dims[d]) + break; logical_coord[d] = 0; } } result.nnz = stored; result.is_iso = iso; - format = cJSON_GetStringValue(cJSON_GetObjectItemCaseSensitive(header, "format")); - if (format == NULL) die("header has no format"); + format = + cJSON_GetStringValue(cJSON_GetObjectItemCaseSensitive(header, "format")); + if (format == NULL) + die("header has no format"); if (!strcmp(format, "custom")) { custom = cJSON_GetObjectItemCaseSensitive(header, "custom"); - description = cJSON_Duplicate( - cJSON_GetObjectItemCaseSensitive(custom, "level"), 1); - if (description == NULL) die("custom target requires custom.level"); - cJSON *transpose = cJSON_GetObjectItemCaseSensitive(custom, "transpose"); + description = + cJSON_Duplicate(cJSON_GetObjectItemCaseSensitive(custom, "level"), 1); + if (description == NULL) + die("custom target requires custom.level"); + cJSON* transpose = cJSON_GetObjectItemCaseSensitive(custom, "transpose"); if (transpose != NULL) for (int d = 0; d < result.rank; ++d) result.transpose[d] = target_transpose[d] = (size_t) cJSON_GetNumberValue(cJSON_GetArrayItem(transpose, d)); } else { description = predefined_level(format, target_transpose, result.rank); - if (description == NULL) die("unknown or rank-incompatible target format"); - for (int d = 0; d < result.rank; ++d) result.transpose[d] = target_transpose[d]; + if (description == NULL) + die("unknown or rank-incompatible target format"); + for (int d = 0; d < result.rank; ++d) + result.transpose[d] = target_transpose[d]; } apply_transpose(&entries, source_transpose, target_transpose); bsp_array_t ordered_values = values_in_entry_order(values, &entries, iso); bsp_destroy_array_t(&values); values = ordered_values; - for (int d = 0; d < result.rank; ++d) stored_dims[d] = result.dims[target_transpose[d]]; - root_ptr[0] = 0; root_ptr[1] = stored; + for (int d = 0; d < result.rank; ++d) + stored_dims[d] = result.dims[target_transpose[d]]; + root_ptr[0] = 0; + root_ptr[1] = stored; result.level = build_level(&context, description, &entries, stored_dims, 0, root_ptr, 1, true, values); if (!strcmp(format, "custom")) { - cJSON *user = cJSON_CreateObject(); + cJSON* user = cJSON_CreateObject(); if (bsp_write_tensor(argv[5], result, NULL, user, 0) != BSP_SUCCESS) die("cannot write Binsparse tensor"); cJSON_Delete(user); @@ -146,8 +173,14 @@ int main(int argc, char **argv) { bsp_destroy_tensor_t(result); free_entries(&entries); - free(logical_coord); free(source_transpose); free(target_transpose); free(stored_dims); - cJSON_Delete(description); cJSON_Delete(header); - cnpy_c_destroy(&dense); cnpy_c_destroy(&pattern); cnpy_c_destroy(&fill); + free(logical_coord); + free(source_transpose); + free(target_transpose); + free(stored_dims); + cJSON_Delete(description); + cJSON_Delete(header); + cnpy_c_destroy(&dense); + cnpy_c_destroy(&pattern); + cnpy_c_destroy(&fill); return 0; } diff --git a/test/compliance/reformat.c b/test/compliance/reformat.c index 26c08e4..48c8375 100644 --- a/test/compliance/reformat.c +++ b/test/compliance/reformat.c @@ -29,13 +29,15 @@ #include #include -bsp_type_t type_from_name(const char *name) { +bsp_type_t type_from_name(const char* name) { size_t length; char base[32]; - if (name == NULL) return BSP_INVALID_TYPE; + if (name == NULL) + return BSP_INVALID_TYPE; length = strlen(name); if (length > 5 && !strncmp(name, "iso[", 4) && name[length - 1] == ']') { - if (length - 5 >= sizeof(base)) return BSP_INVALID_TYPE; + if (length - 5 >= sizeof(base)) + return BSP_INVALID_TYPE; memcpy(base, name + 4, length - 5); base[length - 5] = '\0'; name = base; @@ -46,79 +48,86 @@ bsp_type_t type_from_name(const char *name) { return BSP_INVALID_TYPE; } -static bsp_type_t requested_type(const reformat_context_t *context, - const char *key, bsp_type_t fallback) { - const char *name = cJSON_GetStringValue( +static bsp_type_t requested_type(const reformat_context_t* context, + const char* key, bsp_type_t fallback) { + const char* name = cJSON_GetStringValue( cJSON_GetObjectItemCaseSensitive(context->data_types, key)); bsp_type_t type = type_from_name(name); return type == BSP_INVALID_TYPE ? fallback : type; } -void die(const char *message) { +void die(const char* message) { fprintf(stderr, "reformat: %s\n", message); exit(EXIT_FAILURE); } -cJSON *read_json(const char *path) { - FILE *file = fopen(path, "rb"); +cJSON* read_json(const char* path) { + FILE* file = fopen(path, "rb"); long size = 0; - char *text; - cJSON *json; - if (file == NULL) die(strerror(errno)); + char* text; + cJSON* json; + if (file == NULL) + die(strerror(errno)); if (fseek(file, 0, SEEK_END) != 0 || (size = ftell(file)) < 0 || fseek(file, 0, SEEK_SET) != 0) die("cannot read target header"); text = malloc((size_t) size + 1); - if (text == NULL) die("out of memory"); + if (text == NULL) + die("out of memory"); if (fread(text, 1, (size_t) size, file) != (size_t) size) die("cannot read target header"); fclose(file); text[size] = '\0'; json = cJSON_Parse(text); free(text); - if (!cJSON_IsObject(json)) die("target header must be a JSON object"); + if (!cJSON_IsObject(json)) + die("target header must be a JSON object"); return json; } -cJSON *input_header(const char *path) { +cJSON* input_header(const char* path) { hid_t file = H5Fopen(path, H5F_ACC_RDONLY, H5P_DEFAULT); - char *text = NULL; + char* text = NULL; cJSON *outer, *header; if (file < 0 || bsp_read_attribute(&text, file, "binsparse") != BSP_SUCCESS) die("cannot read input Binsparse header"); H5Fclose(file); outer = cJSON_Parse(text); free(text); - if (!cJSON_IsObject(outer)) die("invalid input Binsparse header"); + if (!cJSON_IsObject(outer)) + die("invalid input Binsparse header"); header = cJSON_DetachItemFromObjectCaseSensitive(outer, "binsparse"); cJSON_Delete(outer); - if (!cJSON_IsObject(header)) die("input has no binsparse header"); + if (!cJSON_IsObject(header)) + die("input has no binsparse header"); return header; } -bool header_values_are_iso(cJSON *header) { - cJSON *types = cJSON_GetObjectItemCaseSensitive(header, "data_types"); - const char *value = cJSON_GetStringValue( - cJSON_GetObjectItemCaseSensitive(types, "values")); +bool header_values_are_iso(cJSON* header) { + cJSON* types = cJSON_GetObjectItemCaseSensitive(header, "data_types"); + const char* value = + cJSON_GetStringValue(cJSON_GetObjectItemCaseSensitive(types, "values")); return value != NULL && strncmp(value, "iso[", 4) == 0; } -bool header_has_fill(cJSON *header) { +bool header_has_fill(cJSON* header) { return cJSON_IsTrue(cJSON_GetObjectItemCaseSensitive(header, "fill")); } -static bsp_error_t read_fill_value(const char *path, bsp_array_t *value) { +static bsp_error_t read_fill_value(const char* path, bsp_array_t* value) { hid_t file = H5Fopen(path, H5F_ACC_RDONLY, H5P_DEFAULT); bsp_error_t error; - if (file < 0) return BSP_ERROR_IO; + if (file < 0) + return BSP_ERROR_IO; error = bsp_read_array(value, file, "fill_value"); H5Fclose(file); return error; } -/* Preserve every supplied partial-header field and add its fill-value buffer. */ -void complete_output(const char *path, cJSON *requested, - const bsp_array_t *fill_value) { +/* Preserve every supplied partial-header field and add its fill-value buffer. + */ +void complete_output(const char* path, cJSON* requested, + const bsp_array_t* fill_value) { hid_t file = H5Fopen(path, H5F_ACC_RDWR, H5P_DEFAULT); char *text = NULL, *updated; cJSON *outer, *header, *item; @@ -128,7 +137,7 @@ void complete_output(const char *path, cJSON *requested, free(text); header = cJSON_GetObjectItemCaseSensitive(outer, "binsparse"); cJSON_ArrayForEach(item, requested) { - cJSON *copy = cJSON_Duplicate(item, 1); + cJSON* copy = cJSON_Duplicate(item, 1); if (cJSON_HasObjectItem(header, item->string)) cJSON_ReplaceItemInObjectCaseSensitive(header, item->string, copy); else @@ -147,41 +156,45 @@ void complete_output(const char *path, cJSON *requested, H5Fclose(file); } -void push_entry(entries_t *entries, const size_t *coord, size_t value) { - entry_t *item; +void push_entry(entries_t* entries, const size_t* coord, size_t value) { + entry_t* item; if (entries->size == entries->capacity) { size_t capacity = entries->capacity == 0 ? 16 : 2 * entries->capacity; - entry_t *grown = realloc(entries->entry, capacity * sizeof(*grown)); - if (grown == NULL) die("out of memory"); + entry_t* grown = realloc(entries->entry, capacity * sizeof(*grown)); + if (grown == NULL) + die("out of memory"); entries->entry = grown; entries->capacity = capacity; } item = &entries->entry[entries->size++]; item->coord = malloc((entries->rank ? (size_t) entries->rank : 1) * sizeof(*item->coord)); - if (item->coord == NULL) die("out of memory"); + if (item->coord == NULL) + die("out of memory"); memcpy(item->coord, coord, (size_t) entries->rank * sizeof(*coord)); item->value = value; } -void free_entries(entries_t *entries) { - for (size_t i = 0; i < entries->size; ++i) free(entries->entry[i].coord); +void free_entries(entries_t* entries) { + for (size_t i = 0; i < entries->size; ++i) + free(entries->entry[i].coord); free(entries->entry); } /* Expand one custom level. begin/end identify the parent positions entering * the level; coordinates are stored row-wise in coords. */ -static size_t stored_dimension(const bsp_tensor_t *tensor, size_t depth) { +static size_t stored_dimension(const bsp_tensor_t* tensor, size_t depth) { size_t logical = tensor->transpose ? tensor->transpose[depth] : depth; - if (logical >= (size_t) tensor->rank) die("transpose is not a permutation"); + if (logical >= (size_t) tensor->rank) + die("transpose is not a permutation"); return tensor->dims[logical]; } -static void level_to_coo(const bsp_level_t *level, const bsp_tensor_t *tensor, - size_t depth, size_t parents, const size_t *coords, - entries_t *out) { +static void level_to_coo(const bsp_level_t* level, const bsp_tensor_t* tensor, + size_t depth, size_t parents, const size_t* coords, + entries_t* out) { if (level->kind == BSP_TENSOR_ELEMENT) { - const bsp_element_t *element = level->data; + const bsp_element_t* element = level->data; if (parents != tensor->nnz) die("element positions do not match number_of_stored_values"); for (size_t p = 0; p < parents; ++p) @@ -192,19 +205,19 @@ static void level_to_coo(const bsp_level_t *level, const bsp_tensor_t *tensor, } if (level->kind == BSP_TENSOR_DENSE) { - const bsp_dense_t *dense = level->data; + const bsp_dense_t* dense = level->data; size_t width = 1, count, *next; for (int d = 0; d < dense->rank; ++d) width *= stored_dimension(tensor, depth + (size_t) d); count = parents * width; next = calloc(count * (size_t) tensor->rank, sizeof(*next)); - if (next == NULL && count != 0) die("out of memory"); + if (next == NULL && count != 0) + die("out of memory"); for (size_t p = 0; p < parents; ++p) { for (size_t suffix = 0; suffix < width; ++suffix) { size_t q = p * width + suffix, n = suffix; memcpy(next + q * (size_t) tensor->rank, - coords + p * (size_t) tensor->rank, - depth * sizeof(*next)); + coords + p * (size_t) tensor->rank, depth * sizeof(*next)); for (int d = dense->rank - 1; d >= 0; --d) { size_t dim = stored_dimension(tensor, depth + (size_t) d); next[q * (size_t) tensor->rank + depth + (size_t) d] = n % dim; @@ -219,11 +232,12 @@ static void level_to_coo(const bsp_level_t *level, const bsp_tensor_t *tensor, } if (level->kind == BSP_TENSOR_SPARSE) { - const bsp_sparse_t *sparse = level->data; + const bsp_sparse_t* sparse = level->data; size_t count = sparse->indices[0].size, *next; size_t q = 0; next = calloc(count * (size_t) tensor->rank, sizeof(*next)); - if (next == NULL && count != 0) die("out of memory"); + if (next == NULL && count != 0) + die("out of memory"); for (size_t p = 0; p < parents; ++p) { size_t begin = 0, end = count; if (sparse->pointers_to != NULL) { @@ -233,8 +247,7 @@ static void level_to_coo(const bsp_level_t *level, const bsp_tensor_t *tensor, } for (size_t i = begin; i < end; ++i, ++q) { memcpy(next + q * (size_t) tensor->rank, - coords + p * (size_t) tensor->rank, - depth * sizeof(*next)); + coords + p * (size_t) tensor->rank, depth * sizeof(*next)); for (int d = 0; d < sparse->rank; ++d) { size_t index = 0; bsp_array_read(sparse->indices[d], i, index); @@ -244,7 +257,8 @@ static void level_to_coo(const bsp_level_t *level, const bsp_tensor_t *tensor, } } } - if (q != count) die("invalid sparse pointers"); + if (q != count) + die("invalid sparse pointers"); level_to_coo(sparse->child, tensor, depth + (size_t) sparse->rank, count, next, out); free(next); @@ -253,18 +267,19 @@ static void level_to_coo(const bsp_level_t *level, const bsp_tensor_t *tensor, die("unknown tensor level"); } -entries_t tensor_to_coo(const bsp_tensor_t *tensor) { +entries_t tensor_to_coo(const bsp_tensor_t* tensor) { entries_t result = {0}; - size_t *root; + size_t* root; result.rank = tensor->rank; root = calloc(tensor->rank ? (size_t) tensor->rank : 1, sizeof(*root)); - if (root == NULL) die("out of memory"); + if (root == NULL) + die("out of memory"); level_to_coo(tensor->level, tensor, 0, 1, root, &result); free(root); return result; } -entries_t matrix_to_coo(bsp_matrix_t *matrix) { +entries_t matrix_to_coo(bsp_matrix_t* matrix) { entries_t result = {0}; result.rank = bsp_matrix_format_is_vector(matrix->format) ? 1 : 2; if (matrix->format == BSP_COOR || matrix->format == BSP_COOC || @@ -272,7 +287,8 @@ entries_t matrix_to_coo(bsp_matrix_t *matrix) { for (size_t i = 0; i < matrix->nnz; ++i) { size_t coord[2] = {0, 0}; bsp_array_read(matrix->indices_0, i, coord[0]); - if (result.rank == 2) bsp_array_read(matrix->indices_1, i, coord[1]); + if (result.rank == 2) + bsp_array_read(matrix->indices_1, i, coord[1]); push_entry(&result, coord, matrix->is_iso ? 0 : i); } return result; @@ -308,14 +324,16 @@ entries_t matrix_to_coo(bsp_matrix_t *matrix) { if (matrix->format == BSP_DVEC || matrix->format == BSP_DMAT || matrix->format == BSP_DMATC) { size_t rows = matrix->format == BSP_DMATC ? matrix->ncols : matrix->nrows; - size_t cols = result.rank == 1 ? 1 - : (matrix->format == BSP_DMATC - ? matrix->nrows - : matrix->ncols); + size_t cols = + result.rank == 1 + ? 1 + : (matrix->format == BSP_DMATC ? matrix->nrows : matrix->ncols); for (size_t i = 0; i < matrix->nnz; ++i) { size_t coord[2] = {i / cols, i % cols}; - if (result.rank == 1) coord[0] = i; - if (coord[0] >= rows) die("dense value count exceeds shape"); + if (result.rank == 1) + coord[0] = i; + if (coord[0] >= rows) + die("dense value count exceeds shape"); push_entry(&result, coord, matrix->is_iso ? 0 : i); } return result; @@ -325,12 +343,12 @@ entries_t matrix_to_coo(bsp_matrix_t *matrix) { } /* A predefined format is only a named view of a custom level tree. */ -bsp_error_t write_predefined(const char *path, bsp_tensor_t tensor, - const char *format, int compression) { +bsp_error_t write_predefined(const char* path, bsp_tensor_t tensor, + const char* format, int compression) { bsp_matrix_t matrix; - bsp_level_t *level = tensor.level; + bsp_level_t* level = tensor.level; bsp_construct_default_matrix_t(&matrix); - matrix.format = bsp_get_matrix_format((char *) format); + matrix.format = bsp_get_matrix_format((char*) format); matrix.nrows = tensor.dims[0]; matrix.ncols = tensor.rank == 1 ? 1 : tensor.dims[1]; matrix.nnz = tensor.nnz; @@ -338,26 +356,31 @@ bsp_error_t write_predefined(const char *path, bsp_tensor_t tensor, matrix.structure = tensor.structure; matrix.values = bsp_get_tensor_values(tensor); - if (matrix.format == BSP_INVALID_FORMAT) return BSP_ERROR_UNSUPPORTED; + if (matrix.format == BSP_INVALID_FORMAT) + return BSP_ERROR_UNSUPPORTED; if (matrix.format == BSP_DVEC || matrix.format == BSP_DMAT || matrix.format == BSP_DMATC) { return bsp_write_matrix(path, matrix, NULL, NULL, compression); } if (matrix.format == BSP_CVEC || matrix.format == BSP_COOR || matrix.format == BSP_COOC) { - bsp_sparse_t *sparse; - if (level->kind != BSP_TENSOR_SPARSE) return BSP_ERROR_INTERNAL; + bsp_sparse_t* sparse; + if (level->kind != BSP_TENSOR_SPARSE) + return BSP_ERROR_INTERNAL; sparse = level->data; matrix.indices_0 = sparse->indices[0]; - if (sparse->rank == 2) matrix.indices_1 = sparse->indices[1]; + if (sparse->rank == 2) + matrix.indices_1 = sparse->indices[1]; return bsp_write_matrix(path, matrix, NULL, NULL, compression); } if (matrix.format == BSP_CSR || matrix.format == BSP_CSC) { - bsp_dense_t *dense; - bsp_sparse_t *sparse; - if (level->kind != BSP_TENSOR_DENSE) return BSP_ERROR_INTERNAL; + bsp_dense_t* dense; + bsp_sparse_t* sparse; + if (level->kind != BSP_TENSOR_DENSE) + return BSP_ERROR_INTERNAL; dense = level->data; - if (dense->child->kind != BSP_TENSOR_SPARSE) return BSP_ERROR_INTERNAL; + if (dense->child->kind != BSP_TENSOR_SPARSE) + return BSP_ERROR_INTERNAL; sparse = dense->child->data; matrix.pointers_to_1 = *sparse->pointers_to; matrix.indices_1 = sparse->indices[0]; @@ -365,9 +388,11 @@ bsp_error_t write_predefined(const char *path, bsp_tensor_t tensor, } if (matrix.format == BSP_DCSR || matrix.format == BSP_DCSC) { bsp_sparse_t *outer, *inner; - if (level->kind != BSP_TENSOR_SPARSE) return BSP_ERROR_INTERNAL; + if (level->kind != BSP_TENSOR_SPARSE) + return BSP_ERROR_INTERNAL; outer = level->data; - if (outer->child->kind != BSP_TENSOR_SPARSE) return BSP_ERROR_INTERNAL; + if (outer->child->kind != BSP_TENSOR_SPARSE) + return BSP_ERROR_INTERNAL; inner = outer->child->data; matrix.indices_0 = outer->indices[0]; matrix.pointers_to_1 = *inner->pointers_to; @@ -378,27 +403,32 @@ bsp_error_t write_predefined(const char *path, bsp_tensor_t tensor, } static int compare_rank; -static int compare_entry(const void *a_, const void *b_) { +static int compare_entry(const void* a_, const void* b_) { const entry_t *a = a_, *b = b_; for (int d = 0; d < compare_rank; ++d) { - if (a->coord[d] < b->coord[d]) return -1; - if (a->coord[d] > b->coord[d]) return 1; + if (a->coord[d] < b->coord[d]) + return -1; + if (a->coord[d] > b->coord[d]) + return 1; } return 0; } -void apply_transpose(entries_t *entries, const size_t *source, - const size_t *target) { - size_t *copy = +void apply_transpose(entries_t* entries, const size_t* source, + const size_t* target) { + size_t* copy = malloc((entries->rank ? (size_t) entries->rank : 1) * sizeof(*copy)); - if (copy == NULL) die("out of memory"); + if (copy == NULL) + die("out of memory"); for (size_t k = 0; k < entries->size; ++k) { memcpy(copy, entries->entry[k].coord, (size_t) entries->rank * sizeof(*copy)); for (int td = 0; td < entries->rank; ++td) { int sd = 0; - while (sd < entries->rank && source[sd] != target[td]) ++sd; - if (sd == entries->rank) die("transpose is not a permutation"); + while (sd < entries->rank && source[sd] != target[td]) + ++sd; + if (sd == entries->rank) + die("transpose is not a permutation"); entries->entry[k].coord[td] = copy[sd]; } } @@ -408,58 +438,62 @@ void apply_transpose(entries_t *entries, const size_t *source, } bsp_array_t values_in_entry_order(const bsp_array_t source, - const entries_t *entries, bool iso) { + const entries_t* entries, bool iso) { bsp_array_t values; size_t size = iso && entries->size != 0 ? 1 : entries->size; if (bsp_construct_array_t(&values, size, source.type) != BSP_SUCCESS) die("out of memory"); for (size_t i = 0; i < size; ++i) - memcpy((char *) values.data + i * bsp_type_size(values.type), - (char *) source.data + entries->entry[iso ? 0 : i].value * - bsp_type_size(source.type), + memcpy((char*) values.data + i * bsp_type_size(values.type), + (char*) source.data + + entries->entry[iso ? 0 : i].value * bsp_type_size(source.type), bsp_type_size(source.type)); return values; } -static size_t prefix_equal(const entry_t *a, const entry_t *b, size_t depth, +static size_t prefix_equal(const entry_t* a, const entry_t* b, size_t depth, int rank) { for (int d = 0; d < rank; ++d) - if (a->coord[depth + (size_t) d] != b->coord[depth + (size_t) d]) return 0; + if (a->coord[depth + (size_t) d] != b->coord[depth + (size_t) d]) + return 0; return 1; } /* Recursively build a level. parent_ptr partitions entries into incoming * fibers. The entries are already lexicographically sorted. */ -bsp_level_t *build_level(const reformat_context_t *context, cJSON *description, - const entries_t *entries, const size_t *dims, - size_t depth, const size_t *parent_ptr, size_t parents, +bsp_level_t* build_level(const reformat_context_t* context, cJSON* description, + const entries_t* entries, const size_t* dims, + size_t depth, const size_t* parent_ptr, size_t parents, bool root, bsp_array_t values) { - cJSON *desc = cJSON_GetObjectItemCaseSensitive(description, "level_desc"); - bsp_level_t *level = calloc(1, sizeof(*level)); - const char *name = cJSON_GetStringValue(desc); - if (level == NULL) die("out of memory"); + cJSON* desc = cJSON_GetObjectItemCaseSensitive(description, "level_desc"); + bsp_level_t* level = calloc(1, sizeof(*level)); + const char* name = cJSON_GetStringValue(desc); + if (level == NULL) + die("out of memory"); if (name != NULL && strcmp(name, "element") == 0) { - bsp_element_t *element = malloc(sizeof(*element)); + bsp_element_t* element = malloc(sizeof(*element)); for (size_t p = 0; p < parents; ++p) if (parent_ptr[p + 1] - parent_ptr[p] != 1) die("each element position must have one value"); - if (element == NULL) die("out of memory"); + if (element == NULL) + die("out of memory"); element->values = values; level->kind = BSP_TENSOR_ELEMENT; level->data = element; return level; } - cJSON *rank_json = cJSON_GetObjectItemCaseSensitive(description, "rank"); + cJSON* rank_json = cJSON_GetObjectItemCaseSensitive(description, "rank"); int rank = (int) cJSON_GetNumberValue(rank_json); - cJSON *child_desc = cJSON_GetObjectItemCaseSensitive(description, "level"); + cJSON* child_desc = cJSON_GetObjectItemCaseSensitive(description, "level"); if (rank <= 0 || depth + (size_t) rank > (size_t) entries->rank || !cJSON_IsObject(child_desc)) die("invalid target level descriptor"); if (strcmp(name ? name : "", "sparse") == 0) { - bsp_sparse_t *sparse = calloc(1, sizeof(*sparse)); + bsp_sparse_t* sparse = calloc(1, sizeof(*sparse)); size_t groups = 0, *child_ptr; - if (sparse == NULL) die("out of memory"); + if (sparse == NULL) + die("out of memory"); for (size_t p = 0; p < parents; ++p) for (size_t i = parent_ptr[p]; i < parent_ptr[p + 1]; ++i) if (i == parent_ptr[p] || @@ -469,15 +503,16 @@ bsp_level_t *build_level(const reformat_context_t *context, cJSON *description, sparse->rank = rank; sparse->indices = calloc((size_t) rank, sizeof(*sparse->indices)); child_ptr = malloc((groups + 1) * sizeof(*child_ptr)); - if (sparse->indices == NULL || child_ptr == NULL) die("out of memory"); + if (sparse->indices == NULL || child_ptr == NULL) + die("out of memory"); for (int d = 0; d < rank; ++d) { char key[64]; snprintf(key, sizeof(key), "indices_%zu", depth + (size_t) d); if (bsp_construct_array_t( &sparse->indices[d], groups, requested_type(context, key, - bsp_pick_integer_type(dims[depth + (size_t) d]))) != - BSP_SUCCESS) + bsp_pick_integer_type( + dims[depth + (size_t) d]))) != BSP_SUCCESS) die("out of memory"); } if (!root) { @@ -485,9 +520,9 @@ bsp_level_t *build_level(const reformat_context_t *context, cJSON *description, snprintf(key, sizeof(key), "pointers_to_%zu", depth); sparse->pointers_to = malloc(sizeof(*sparse->pointers_to)); if (sparse->pointers_to == NULL || - bsp_construct_array_t(sparse->pointers_to, parents + 1, - requested_type(context, key, - bsp_pick_integer_type(groups))) != + bsp_construct_array_t( + sparse->pointers_to, parents + 1, + requested_type(context, key, bsp_pick_integer_type(groups))) != BSP_SUCCESS) die("out of memory"); } @@ -502,7 +537,8 @@ bsp_level_t *build_level(const reformat_context_t *context, cJSON *description, if (i == parent_ptr[p] || !prefix_equal(&entries->entry[i - 1], &entries->entry[i], depth, rank)) { - if (groups != 0) child_ptr[groups] = i; + if (groups != 0) + child_ptr[groups] = i; for (int d = 0; d < rank; ++d) bsp_array_write(sparse->indices[d], groups, entries->entry[i].coord[depth + (size_t) d]); @@ -515,9 +551,9 @@ bsp_level_t *build_level(const reformat_context_t *context, cJSON *description, bsp_array_t pointers = *sparse->pointers_to; bsp_array_write(pointers, parents, groups); } - sparse->child = build_level(context, child_desc, entries, dims, - depth + (size_t) rank, child_ptr, groups, false, - values); + sparse->child = + build_level(context, child_desc, entries, dims, depth + (size_t) rank, + child_ptr, groups, false, values); free(child_ptr); level->kind = BSP_TENSOR_SPARSE; level->data = sparse; @@ -525,13 +561,16 @@ bsp_level_t *build_level(const reformat_context_t *context, cJSON *description, } if (strcmp(name ? name : "", "dense") == 0) { - bsp_dense_t *dense = malloc(sizeof(*dense)); + bsp_dense_t* dense = malloc(sizeof(*dense)); size_t width = 1, children, *child_ptr; - if (dense == NULL) die("out of memory"); - for (int d = 0; d < rank; ++d) width *= dims[depth + (size_t) d]; + if (dense == NULL) + die("out of memory"); + for (int d = 0; d < rank; ++d) + width *= dims[depth + (size_t) d]; children = parents * width; child_ptr = malloc((children + 1) * sizeof(*child_ptr)); - if (child_ptr == NULL) die("out of memory"); + if (child_ptr == NULL) + die("out of memory"); size_t at = 0; for (size_t p = 0; p < parents; ++p) { size_t end = parent_ptr[p + 1]; @@ -546,16 +585,17 @@ bsp_level_t *build_level(const reformat_context_t *context, cJSON *description, equal = 0; n /= dim; } - if (!equal) break; + if (!equal) + break; ++at; } } } child_ptr[children] = entries->size; dense->rank = rank; - dense->child = build_level(context, child_desc, entries, dims, - depth + (size_t) rank, child_ptr, children, - false, values); + dense->child = + build_level(context, child_desc, entries, dims, depth + (size_t) rank, + child_ptr, children, false, values); free(child_ptr); level->kind = BSP_TENSOR_DENSE; level->data = dense; @@ -565,24 +605,54 @@ bsp_level_t *build_level(const reformat_context_t *context, cJSON *description, return NULL; } -cJSON *predefined_level(const char *format, size_t *transpose, int rank) { - const char *json = NULL; - if (!strcmp(format, "DVEC")) json = "{\"level_desc\":\"dense\",\"rank\":1,\"level\":{\"level_desc\":\"element\"}}"; - else if (!strcmp(format, "CVEC")) json = "{\"level_desc\":\"sparse\",\"rank\":1,\"level\":{\"level_desc\":\"element\"}}"; - else if (!strcmp(format, "DMAT") || !strcmp(format, "DMATR")) json = "{\"level_desc\":\"dense\",\"rank\":2,\"level\":{\"level_desc\":\"element\"}}"; - else if (!strcmp(format, "DMATC")) { json = "{\"level_desc\":\"dense\",\"rank\":2,\"level\":{\"level_desc\":\"element\"}}"; transpose[0]=1; transpose[1]=0; } - else if (!strcmp(format, "COO") || !strcmp(format, "COOR")) json = "{\"level_desc\":\"sparse\",\"rank\":2,\"level\":{\"level_desc\":\"element\"}}"; - else if (!strcmp(format, "COOC")) { json = "{\"level_desc\":\"sparse\",\"rank\":2,\"level\":{\"level_desc\":\"element\"}}"; transpose[0]=1; transpose[1]=0; } - else if (!strcmp(format, "CSR")) json = "{\"level_desc\":\"dense\",\"rank\":1,\"level\":{\"level_desc\":\"sparse\",\"rank\":1,\"level\":{\"level_desc\":\"element\"}}}"; - else if (!strcmp(format, "CSC")) { json = "{\"level_desc\":\"dense\",\"rank\":1,\"level\":{\"level_desc\":\"sparse\",\"rank\":1,\"level\":{\"level_desc\":\"element\"}}}"; transpose[0]=1; transpose[1]=0; } - else if (!strcmp(format, "DCSR")) json = "{\"level_desc\":\"sparse\",\"rank\":1,\"level\":{\"level_desc\":\"sparse\",\"rank\":1,\"level\":{\"level_desc\":\"element\"}}}"; - else if (!strcmp(format, "DCSC")) { json = "{\"level_desc\":\"sparse\",\"rank\":1,\"level\":{\"level_desc\":\"sparse\",\"rank\":1,\"level\":{\"level_desc\":\"element\"}}}"; transpose[0]=1; transpose[1]=0; } - if (json == NULL || (rank != 1 && rank != 2)) return NULL; +cJSON* predefined_level(const char* format, size_t* transpose, int rank) { + const char* json = NULL; + if (!strcmp(format, "DVEC")) + json = "{\"level_desc\":\"dense\",\"rank\":1,\"level\":{\"level_desc\":" + "\"element\"}}"; + else if (!strcmp(format, "CVEC")) + json = "{\"level_desc\":\"sparse\",\"rank\":1,\"level\":{\"level_desc\":" + "\"element\"}}"; + else if (!strcmp(format, "DMAT") || !strcmp(format, "DMATR")) + json = "{\"level_desc\":\"dense\",\"rank\":2,\"level\":{\"level_desc\":" + "\"element\"}}"; + else if (!strcmp(format, "DMATC")) { + json = "{\"level_desc\":\"dense\",\"rank\":2,\"level\":{\"level_desc\":" + "\"element\"}}"; + transpose[0] = 1; + transpose[1] = 0; + } else if (!strcmp(format, "COO") || !strcmp(format, "COOR")) + json = "{\"level_desc\":\"sparse\",\"rank\":2,\"level\":{\"level_desc\":" + "\"element\"}}"; + else if (!strcmp(format, "COOC")) { + json = "{\"level_desc\":\"sparse\",\"rank\":2,\"level\":{\"level_desc\":" + "\"element\"}}"; + transpose[0] = 1; + transpose[1] = 0; + } else if (!strcmp(format, "CSR")) + json = "{\"level_desc\":\"dense\",\"rank\":1,\"level\":{\"level_desc\":" + "\"sparse\",\"rank\":1,\"level\":{\"level_desc\":\"element\"}}}"; + else if (!strcmp(format, "CSC")) { + json = "{\"level_desc\":\"dense\",\"rank\":1,\"level\":{\"level_desc\":" + "\"sparse\",\"rank\":1,\"level\":{\"level_desc\":\"element\"}}}"; + transpose[0] = 1; + transpose[1] = 0; + } else if (!strcmp(format, "DCSR")) + json = "{\"level_desc\":\"sparse\",\"rank\":1,\"level\":{\"level_desc\":" + "\"sparse\",\"rank\":1,\"level\":{\"level_desc\":\"element\"}}}"; + else if (!strcmp(format, "DCSC")) { + json = "{\"level_desc\":\"sparse\",\"rank\":1,\"level\":{\"level_desc\":" + "\"sparse\",\"rank\":1,\"level\":{\"level_desc\":\"element\"}}}"; + transpose[0] = 1; + transpose[1] = 0; + } + if (json == NULL || (rank != 1 && rank != 2)) + return NULL; return cJSON_Parse(json); } -bsp_error_t bsp_reformat_file(const char *input, const char *output, - cJSON *target_header, int compression) { +bsp_error_t bsp_reformat_file(const char* input, const char* output, + cJSON* target_header, int compression) { cJSON *source_header, *format_json, *custom, *description; reformat_context_t context; const char *source_format, *target_format; @@ -603,8 +673,10 @@ bsp_error_t bsp_reformat_file(const char *input, const char *output, cJSON_GetObjectItemCaseSensitive(target_header, "data_types"); format_json = cJSON_GetObjectItemCaseSensitive(source_header, "format"); source_format = cJSON_GetStringValue(format_json); - target_format = cJSON_GetStringValue(cJSON_GetObjectItemCaseSensitive(target_header, "format")); - if (source_format == NULL || target_format == NULL) die("header has no format string"); + target_format = cJSON_GetStringValue( + cJSON_GetObjectItemCaseSensitive(target_header, "format")); + if (source_format == NULL || target_format == NULL) + die("header has no format string"); if (header_has_fill(source_header)) { if (read_fill_value(input, &fill_value) != BSP_SUCCESS) die("cannot read input fill_value"); @@ -614,7 +686,8 @@ bsp_error_t bsp_reformat_file(const char *input, const char *output, if (source_is_tensor) { tensor = bsp_read_tensor(input, NULL); - if (tensor.level == NULL) die("failed to parse input tensor"); + if (tensor.level == NULL) + die("failed to parse input tensor"); tensor.is_iso = header_values_are_iso(source_header); rank = tensor.rank; source_values = bsp_get_tensor_values(tensor); @@ -633,9 +706,11 @@ bsp_error_t bsp_reformat_file(const char *input, const char *output, source_transpose = malloc(allocated_rank * sizeof(*source_transpose)); target_transpose = malloc(allocated_rank * sizeof(*target_transpose)); stored_dims = malloc(allocated_rank * sizeof(*stored_dims)); - if (!source_transpose || !target_transpose || !stored_dims) die("out of memory"); + if (!source_transpose || !target_transpose || !stored_dims) + die("out of memory"); for (int d = 0; d < rank; ++d) { - source_transpose[d] = source_is_tensor && tensor.transpose ? tensor.transpose[d] : (size_t) d; + source_transpose[d] = + source_is_tensor && tensor.transpose ? tensor.transpose[d] : (size_t) d; target_transpose[d] = (size_t) d; } if (!source_is_tensor && rank == 2 && @@ -648,24 +723,29 @@ bsp_error_t bsp_reformat_file(const char *input, const char *output, if (!strcmp(target_format, "custom")) { custom = cJSON_GetObjectItemCaseSensitive(target_header, "custom"); description = cJSON_GetObjectItemCaseSensitive(custom, "level"); - cJSON *transpose = cJSON_GetObjectItemCaseSensitive(custom, "transpose"); - if (!cJSON_IsObject(description)) die("custom target requires custom.level"); + cJSON* transpose = cJSON_GetObjectItemCaseSensitive(custom, "transpose"); + if (!cJSON_IsObject(description)) + die("custom target requires custom.level"); if (transpose != NULL) { - if (cJSON_GetArraySize(transpose) != rank) die("transpose rank mismatch"); + if (cJSON_GetArraySize(transpose) != rank) + die("transpose rank mismatch"); for (int d = 0; d < rank; ++d) - target_transpose[d] = (size_t) cJSON_GetNumberValue(cJSON_GetArrayItem(transpose, d)); + target_transpose[d] = + (size_t) cJSON_GetNumberValue(cJSON_GetArrayItem(transpose, d)); } description = cJSON_Duplicate(description, 1); } else { description = predefined_level(target_format, target_transpose, rank); - if (description == NULL) die("unknown or rank-incompatible target format"); + if (description == NULL) + die("unknown or rank-incompatible target format"); } apply_transpose(&entries, source_transpose, target_transpose); for (int d = 0; d < rank; ++d) { size_t logical = target_transpose[d]; - stored_dims[d] = source_is_tensor ? tensor.dims[logical] - : (logical == 0 ? matrix.nrows : matrix.ncols); + stored_dims[d] = source_is_tensor + ? tensor.dims[logical] + : (logical == 0 ? matrix.nrows : matrix.ncols); } values = values_in_entry_order(source_values, &entries, iso); bsp_tensor_t result = bsp_construct_default_tensor_t(); @@ -675,17 +755,21 @@ bsp_error_t bsp_reformat_file(const char *input, const char *output, result.structure = source_is_tensor ? tensor.structure : matrix.structure; result.dims = malloc(allocated_rank * sizeof(*result.dims)); result.transpose = malloc(allocated_rank * sizeof(*result.transpose)); - if (!result.dims || !result.transpose) die("out of memory"); + if (!result.dims || !result.transpose) + die("out of memory"); for (int d = 0; d < rank; ++d) { - result.dims[d] = source_is_tensor ? tensor.dims[d] : (d == 0 ? matrix.nrows : matrix.ncols); + result.dims[d] = source_is_tensor ? tensor.dims[d] + : (d == 0 ? matrix.nrows : matrix.ncols); result.transpose[d] = target_transpose[d]; } - root_ptr[0] = 0; root_ptr[1] = entries.size; + root_ptr[0] = 0; + root_ptr[1] = entries.size; result.level = build_level(&context, description, &entries, stored_dims, 0, root_ptr, 1, true, values); if (!strcmp(target_format, "custom")) { - cJSON *user = cJSON_CreateObject(); - if (bsp_write_tensor(output, result, NULL, user, compression) != BSP_SUCCESS) + cJSON* user = cJSON_CreateObject(); + if (bsp_write_tensor(output, result, NULL, user, compression) != + BSP_SUCCESS) die("failed to write output tensor"); cJSON_Delete(user); } else if (write_predefined(output, result, target_format, compression) != @@ -696,9 +780,16 @@ bsp_error_t bsp_reformat_file(const char *input, const char *output, bsp_destroy_tensor_t(result); free_entries(&entries); - if (source_is_tensor) bsp_destroy_tensor_t(tensor); else bsp_destroy_matrix_t(&matrix); - if (owns_fill) bsp_destroy_array_t(&fill_value); - free(source_transpose); free(target_transpose); free(stored_dims); - cJSON_Delete(description); cJSON_Delete(source_header); + if (source_is_tensor) + bsp_destroy_tensor_t(tensor); + else + bsp_destroy_matrix_t(&matrix); + if (owns_fill) + bsp_destroy_array_t(&fill_value); + free(source_transpose); + free(target_transpose); + free(stored_dims); + cJSON_Delete(description); + cJSON_Delete(source_header); return BSP_SUCCESS; } diff --git a/test/compliance/reformat.h b/test/compliance/reformat.h index ac2a10e..9348af2 100644 --- a/test/compliance/reformat.h +++ b/test/compliance/reformat.h @@ -1,3 +1,9 @@ +/* + * SPDX-FileCopyrightText: 2026 Binsparse Developers + * + * SPDX-License-Identifier: BSD-3-Clause + */ + #pragma once #include @@ -11,47 +17,46 @@ #include typedef struct { - size_t *coord; + size_t* coord; size_t value; } entry_t; typedef struct { - entry_t *entry; + entry_t* entry; size_t size; size_t capacity; int rank; } entries_t; typedef struct { - cJSON *data_types; + cJSON* data_types; } reformat_context_t; -void die(const char *message); -cJSON *read_json(const char *path); -cJSON *input_header(const char *path); -bool header_values_are_iso(cJSON *header); -bool header_has_fill(cJSON *header); -void complete_output(const char *path, cJSON *requested, - const bsp_array_t *fill_value); - -bsp_type_t type_from_name(const char *name); -void push_entry(entries_t *entries, const size_t *coord, size_t value); -void free_entries(entries_t *entries); -entries_t tensor_to_coo(const bsp_tensor_t *tensor); -entries_t matrix_to_coo(bsp_matrix_t *matrix); -void apply_transpose(entries_t *entries, const size_t *source, - const size_t *target); +void die(const char* message); +cJSON* read_json(const char* path); +cJSON* input_header(const char* path); +bool header_values_are_iso(cJSON* header); +bool header_has_fill(cJSON* header); +void complete_output(const char* path, cJSON* requested, + const bsp_array_t* fill_value); + +bsp_type_t type_from_name(const char* name); +void push_entry(entries_t* entries, const size_t* coord, size_t value); +void free_entries(entries_t* entries); +entries_t tensor_to_coo(const bsp_tensor_t* tensor); +entries_t matrix_to_coo(bsp_matrix_t* matrix); +void apply_transpose(entries_t* entries, const size_t* source, + const size_t* target); bsp_array_t values_in_entry_order(const bsp_array_t source, - const entries_t *entries, bool iso); - -cJSON *predefined_level(const char *format, size_t *transpose, int rank); -bsp_level_t *build_level(const reformat_context_t *context, - cJSON *description, const entries_t *entries, - const size_t *dims, size_t depth, - const size_t *parent_ptr, size_t parents, bool root, - bsp_array_t values); -bsp_error_t write_predefined(const char *path, bsp_tensor_t tensor, - const char *format, int compression); - -bsp_error_t bsp_reformat_file(const char *input, const char *output, - cJSON *target_header, int compression); + const entries_t* entries, bool iso); + +cJSON* predefined_level(const char* format, size_t* transpose, int rank); +bsp_level_t* build_level(const reformat_context_t* context, cJSON* description, + const entries_t* entries, const size_t* dims, + size_t depth, const size_t* parent_ptr, size_t parents, + bool root, bsp_array_t values); +bsp_error_t write_predefined(const char* path, bsp_tensor_t tensor, + const char* format, int compression); + +bsp_error_t bsp_reformat_file(const char* input, const char* output, + cJSON* target_header, int compression); diff --git a/third_party/cnpy/CMakeLists.txt b/third_party/cnpy/CMakeLists.txt index 9eb550f..6fd1d65 100644 --- a/third_party/cnpy/CMakeLists.txt +++ b/third_party/cnpy/CMakeLists.txt @@ -1,3 +1,9 @@ +#[[ +SPDX-FileCopyrightText: 2011 Carl Rogers + +SPDX-License-Identifier: MIT +]] + CMAKE_MINIMUM_REQUIRED(VERSION 3.0 FATAL_ERROR) if(COMMAND cmake_policy) cmake_policy(SET CMP0003 NEW) diff --git a/third_party/cnpy/README.md b/third_party/cnpy/README.md index 37c4a43..edef9e5 100644 --- a/third_party/cnpy/README.md +++ b/third_party/cnpy/README.md @@ -1,19 +1,25 @@ + + # Purpose: -NumPy offers the `save` method for easy saving of arrays into .npy and `savez` for zipping multiple .npy arrays together into a .npz file. +NumPy offers the `save` method for easy saving of arrays into .npy and `savez` for zipping multiple .npy arrays together into a .npz file. -`cnpy` lets you read and write to these formats in C++. +`cnpy` lets you read and write to these formats in C++. The motivation comes from scientific programming where large amounts of data are generated in C++ and analyzed in Python. -Writing to .npy has the advantage of using low-level C++ I/O (fread and fwrite) for speed and binary format for size. +Writing to .npy has the advantage of using low-level C++ I/O (fread and fwrite) for speed and binary format for size. The .npy file header takes care of specifying the size, shape, and data type of the array, so specifying the format of the data is unnecessary. Loading data written in numpy formats into C++ is equally simple, but requires you to type-cast the loaded data to the type of your choice. # Installation: -Default installation directory is /usr/local. +Default installation directory is /usr/local. To specify a different directory, add `-DCMAKE_INSTALL_PREFIX=/path/to/install/dir` to the cmake invocation in step 4. 1. get [cmake](www.cmake.org) @@ -36,12 +42,12 @@ g++ -o mycode mycode.cpp -L/path/to/install/dir -lcnpy -lz --std=c++11 There are two functions for writing data: `npy_save` and `npz_save`. There are 3 functions for reading: -- `npy_load` will load a .npy file. -- `npz_load(fname)` will load a .npz and return a dictionary of NpyArray structues. +- `npy_load` will load a .npy file. +- `npz_load(fname)` will load a .npz and return a dictionary of NpyArray structues. - `npz_load(fname,varname)` will load and return the NpyArray for data varname from the specified .npz file. -The data structure for loaded data is below. -Data is accessed via the `data()`-method, which returns a pointer of the specified type (which must match the underlying datatype of the data). +The data structure for loaded data is below. +Data is accessed via the `data()`-method, which returns a pointer of the specified type (which must match the underlying datatype of the data). The array shape and word size are read from the npy header. ```c++ diff --git a/third_party/cnpy/cnpy.cpp b/third_party/cnpy/cnpy.cpp index 902730c..b275d5d 100644 --- a/third_party/cnpy/cnpy.cpp +++ b/third_party/cnpy/cnpy.cpp @@ -1,340 +1,386 @@ -//Copyright (C) 2011 Carl Rogers -//Released under MIT License -//license available in LICENSE file, or at http://www.opensource.org/licenses/mit-license.php - -#include"cnpy.h" -#include -#include -#include -#include -#include -#include -#include +// SPDX-FileCopyrightText: 2011 Carl Rogers +// +// SPDX-License-Identifier: MIT + +// Copyright (C) 2011 Carl Rogers +// Released under MIT License +// license available in LICENSE file, or at +// http://www.opensource.org/licenses/mit-license.php + +#include "cnpy.h" +#include +#include +#include +#include +#include #include +#include +#include char cnpy::BigEndianTest() { - int x = 1; - return (((char *)&x)[0]) ? '<' : '>'; + int x = 1; + return (((char*) &x)[0]) ? '<' : '>'; } -char cnpy::map_type(const std::type_info& t) -{ - if(t == typeid(float) ) return 'f'; - if(t == typeid(double) ) return 'f'; - if(t == typeid(long double) ) return 'f'; - - if(t == typeid(int) ) return 'i'; - if(t == typeid(char) ) return 'i'; - if(t == typeid(signed char) ) return 'i'; - if(t == typeid(short) ) return 'i'; - if(t == typeid(long) ) return 'i'; - if(t == typeid(long long) ) return 'i'; - - if(t == typeid(unsigned char) ) return 'u'; - if(t == typeid(unsigned short) ) return 'u'; - if(t == typeid(unsigned long) ) return 'u'; - if(t == typeid(unsigned long long) ) return 'u'; - if(t == typeid(unsigned int) ) return 'u'; - - if(t == typeid(bool) ) return 'b'; - - if(t == typeid(std::complex) ) return 'c'; - if(t == typeid(std::complex) ) return 'c'; - if(t == typeid(std::complex) ) return 'c'; - - else return '?'; +char cnpy::map_type(const std::type_info& t) { + if (t == typeid(float)) + return 'f'; + if (t == typeid(double)) + return 'f'; + if (t == typeid(long double)) + return 'f'; + + if (t == typeid(int)) + return 'i'; + if (t == typeid(char)) + return 'i'; + if (t == typeid(signed char)) + return 'i'; + if (t == typeid(short)) + return 'i'; + if (t == typeid(long)) + return 'i'; + if (t == typeid(long long)) + return 'i'; + + if (t == typeid(unsigned char)) + return 'u'; + if (t == typeid(unsigned short)) + return 'u'; + if (t == typeid(unsigned long)) + return 'u'; + if (t == typeid(unsigned long long)) + return 'u'; + if (t == typeid(unsigned int)) + return 'u'; + + if (t == typeid(bool)) + return 'b'; + + if (t == typeid(std::complex)) + return 'c'; + if (t == typeid(std::complex)) + return 'c'; + if (t == typeid(std::complex)) + return 'c'; + + else + return '?'; } -template<> std::vector& cnpy::operator+=(std::vector& lhs, const std::string rhs) { - lhs.insert(lhs.end(),rhs.begin(),rhs.end()); - return lhs; +template <> +std::vector& cnpy::operator+=(std::vector& lhs, + const std::string rhs) { + lhs.insert(lhs.end(), rhs.begin(), rhs.end()); + return lhs; } -template<> std::vector& cnpy::operator+=(std::vector& lhs, const char* rhs) { - //write in little endian - size_t len = strlen(rhs); - lhs.reserve(len); - for(size_t byte = 0; byte < len; byte++) { - lhs.push_back(rhs[byte]); - } - return lhs; +template <> +std::vector& cnpy::operator+=(std::vector& lhs, const char* rhs) { + // write in little endian + size_t len = strlen(rhs); + lhs.reserve(len); + for (size_t byte = 0; byte < len; byte++) { + lhs.push_back(rhs[byte]); + } + return lhs; } -void cnpy::parse_npy_header(unsigned char* buffer,size_t& word_size, std::vector& shape, bool& fortran_order) { - //std::string magic_string(buffer,6); - uint8_t major_version = *reinterpret_cast(buffer+6); - uint8_t minor_version = *reinterpret_cast(buffer+7); - uint16_t header_len = *reinterpret_cast(buffer+8); - std::string header(reinterpret_cast(buffer+9),header_len); - - size_t loc1, loc2; - - //fortran order - loc1 = header.find("fortran_order")+16; - fortran_order = (header.substr(loc1,4) == "True" ? true : false); - - //shape - loc1 = header.find("("); - loc2 = header.find(")"); - - std::regex num_regex("[0-9][0-9]*"); - std::smatch sm; - shape.clear(); - - std::string str_shape = header.substr(loc1+1,loc2-loc1-1); - while(std::regex_search(str_shape, sm, num_regex)) { - shape.push_back(std::stoi(sm[0].str())); - str_shape = sm.suffix().str(); - } - - //endian, word size, data type - //byte order code | stands for not applicable. - //not sure when this applies except for byte array - loc1 = header.find("descr")+9; - bool littleEndian = (header[loc1] == '<' || header[loc1] == '|' ? true : false); - assert(littleEndian); - - //char type = header[loc1+1]; - //assert(type == map_type(T)); - - std::string str_ws = header.substr(loc1+2); - loc2 = str_ws.find("'"); - word_size = atoi(str_ws.substr(0,loc2).c_str()); +void cnpy::parse_npy_header(unsigned char* buffer, size_t& word_size, + std::vector& shape, bool& fortran_order) { + // std::string magic_string(buffer,6); + uint8_t major_version = *reinterpret_cast(buffer + 6); + uint8_t minor_version = *reinterpret_cast(buffer + 7); + uint16_t header_len = *reinterpret_cast(buffer + 8); + std::string header(reinterpret_cast(buffer + 9), header_len); + + size_t loc1, loc2; + + // fortran order + loc1 = header.find("fortran_order") + 16; + fortran_order = (header.substr(loc1, 4) == "True" ? true : false); + + // shape + loc1 = header.find("("); + loc2 = header.find(")"); + + std::regex num_regex("[0-9][0-9]*"); + std::smatch sm; + shape.clear(); + + std::string str_shape = header.substr(loc1 + 1, loc2 - loc1 - 1); + while (std::regex_search(str_shape, sm, num_regex)) { + shape.push_back(std::stoi(sm[0].str())); + str_shape = sm.suffix().str(); + } + + // endian, word size, data type + // byte order code | stands for not applicable. + // not sure when this applies except for byte array + loc1 = header.find("descr") + 9; + bool littleEndian = + (header[loc1] == '<' || header[loc1] == '|' ? true : false); + assert(littleEndian); + + // char type = header[loc1+1]; + // assert(type == map_type(T)); + + std::string str_ws = header.substr(loc1 + 2); + loc2 = str_ws.find("'"); + word_size = atoi(str_ws.substr(0, loc2).c_str()); } -void cnpy::parse_npy_header(FILE* fp, size_t& word_size, std::vector& shape, bool& fortran_order) { - char buffer[256]; - size_t res = fread(buffer,sizeof(char),11,fp); - if(res != 11) - throw std::runtime_error("parse_npy_header: failed fread"); - std::string header = fgets(buffer,256,fp); - assert(header[header.size()-1] == '\n'); - - size_t loc1, loc2; - - //fortran order - loc1 = header.find("fortran_order"); - if (loc1 == std::string::npos) - throw std::runtime_error("parse_npy_header: failed to find header keyword: 'fortran_order'"); - loc1 += 16; - fortran_order = (header.substr(loc1,4) == "True" ? true : false); - - //shape - loc1 = header.find("("); - loc2 = header.find(")"); - if (loc1 == std::string::npos || loc2 == std::string::npos) - throw std::runtime_error("parse_npy_header: failed to find header keyword: '(' or ')'"); - - std::regex num_regex("[0-9][0-9]*"); - std::smatch sm; - shape.clear(); - - std::string str_shape = header.substr(loc1+1,loc2-loc1-1); - while(std::regex_search(str_shape, sm, num_regex)) { - shape.push_back(std::stoi(sm[0].str())); - str_shape = sm.suffix().str(); - } - - //endian, word size, data type - //byte order code | stands for not applicable. - //not sure when this applies except for byte array - loc1 = header.find("descr"); - if (loc1 == std::string::npos) - throw std::runtime_error("parse_npy_header: failed to find header keyword: 'descr'"); - loc1 += 9; - bool littleEndian = (header[loc1] == '<' || header[loc1] == '|' ? true : false); - assert(littleEndian); - - //char type = header[loc1+1]; - //assert(type == map_type(T)); - - std::string str_ws = header.substr(loc1+2); - loc2 = str_ws.find("'"); - word_size = atoi(str_ws.substr(0,loc2).c_str()); +void cnpy::parse_npy_header(FILE* fp, size_t& word_size, + std::vector& shape, bool& fortran_order) { + char buffer[256]; + size_t res = fread(buffer, sizeof(char), 11, fp); + if (res != 11) + throw std::runtime_error("parse_npy_header: failed fread"); + std::string header = fgets(buffer, 256, fp); + assert(header[header.size() - 1] == '\n'); + + size_t loc1, loc2; + + // fortran order + loc1 = header.find("fortran_order"); + if (loc1 == std::string::npos) + throw std::runtime_error( + "parse_npy_header: failed to find header keyword: 'fortran_order'"); + loc1 += 16; + fortran_order = (header.substr(loc1, 4) == "True" ? true : false); + + // shape + loc1 = header.find("("); + loc2 = header.find(")"); + if (loc1 == std::string::npos || loc2 == std::string::npos) + throw std::runtime_error( + "parse_npy_header: failed to find header keyword: '(' or ')'"); + + std::regex num_regex("[0-9][0-9]*"); + std::smatch sm; + shape.clear(); + + std::string str_shape = header.substr(loc1 + 1, loc2 - loc1 - 1); + while (std::regex_search(str_shape, sm, num_regex)) { + shape.push_back(std::stoi(sm[0].str())); + str_shape = sm.suffix().str(); + } + + // endian, word size, data type + // byte order code | stands for not applicable. + // not sure when this applies except for byte array + loc1 = header.find("descr"); + if (loc1 == std::string::npos) + throw std::runtime_error( + "parse_npy_header: failed to find header keyword: 'descr'"); + loc1 += 9; + bool littleEndian = + (header[loc1] == '<' || header[loc1] == '|' ? true : false); + assert(littleEndian); + + // char type = header[loc1+1]; + // assert(type == map_type(T)); + + std::string str_ws = header.substr(loc1 + 2); + loc2 = str_ws.find("'"); + word_size = atoi(str_ws.substr(0, loc2).c_str()); } -void cnpy::parse_zip_footer(FILE* fp, uint16_t& nrecs, size_t& global_header_size, size_t& global_header_offset) -{ - std::vector footer(22); - fseek(fp,-22,SEEK_END); - size_t res = fread(&footer[0],sizeof(char),22,fp); - if(res != 22) - throw std::runtime_error("parse_zip_footer: failed fread"); - - uint16_t disk_no, disk_start, nrecs_on_disk, comment_len; - disk_no = *(uint16_t*) &footer[4]; - disk_start = *(uint16_t*) &footer[6]; - nrecs_on_disk = *(uint16_t*) &footer[8]; - nrecs = *(uint16_t*) &footer[10]; - global_header_size = *(uint32_t*) &footer[12]; - global_header_offset = *(uint32_t*) &footer[16]; - comment_len = *(uint16_t*) &footer[20]; - - assert(disk_no == 0); - assert(disk_start == 0); - assert(nrecs_on_disk == nrecs); - assert(comment_len == 0); +void cnpy::parse_zip_footer(FILE* fp, uint16_t& nrecs, + size_t& global_header_size, + size_t& global_header_offset) { + std::vector footer(22); + fseek(fp, -22, SEEK_END); + size_t res = fread(&footer[0], sizeof(char), 22, fp); + if (res != 22) + throw std::runtime_error("parse_zip_footer: failed fread"); + + uint16_t disk_no, disk_start, nrecs_on_disk, comment_len; + disk_no = *(uint16_t*) &footer[4]; + disk_start = *(uint16_t*) &footer[6]; + nrecs_on_disk = *(uint16_t*) &footer[8]; + nrecs = *(uint16_t*) &footer[10]; + global_header_size = *(uint32_t*) &footer[12]; + global_header_offset = *(uint32_t*) &footer[16]; + comment_len = *(uint16_t*) &footer[20]; + + assert(disk_no == 0); + assert(disk_start == 0); + assert(nrecs_on_disk == nrecs); + assert(comment_len == 0); } cnpy::NpyArray load_the_npy_file(FILE* fp) { - std::vector shape; - size_t word_size; - bool fortran_order; - cnpy::parse_npy_header(fp,word_size,shape,fortran_order); - - cnpy::NpyArray arr(shape, word_size, fortran_order); - size_t nread = fread(arr.data(),1,arr.num_bytes(),fp); - if(nread != arr.num_bytes()) - throw std::runtime_error("load_the_npy_file: failed fread"); - return arr; + std::vector shape; + size_t word_size; + bool fortran_order; + cnpy::parse_npy_header(fp, word_size, shape, fortran_order); + + cnpy::NpyArray arr(shape, word_size, fortran_order); + size_t nread = fread(arr.data(), 1, arr.num_bytes(), fp); + if (nread != arr.num_bytes()) + throw std::runtime_error("load_the_npy_file: failed fread"); + return arr; } -cnpy::NpyArray load_the_npz_array(FILE* fp, uint32_t compr_bytes, uint32_t uncompr_bytes) { +cnpy::NpyArray load_the_npz_array(FILE* fp, uint32_t compr_bytes, + uint32_t uncompr_bytes) { - std::vector buffer_compr(compr_bytes); - std::vector buffer_uncompr(uncompr_bytes); - size_t nread = fread(&buffer_compr[0],1,compr_bytes,fp); - if(nread != compr_bytes) - throw std::runtime_error("load_the_npy_file: failed fread"); + std::vector buffer_compr(compr_bytes); + std::vector buffer_uncompr(uncompr_bytes); + size_t nread = fread(&buffer_compr[0], 1, compr_bytes, fp); + if (nread != compr_bytes) + throw std::runtime_error("load_the_npy_file: failed fread"); - int err; - z_stream d_stream; + int err; + z_stream d_stream; - d_stream.zalloc = Z_NULL; - d_stream.zfree = Z_NULL; - d_stream.opaque = Z_NULL; - d_stream.avail_in = 0; - d_stream.next_in = Z_NULL; - err = inflateInit2(&d_stream, -MAX_WBITS); + d_stream.zalloc = Z_NULL; + d_stream.zfree = Z_NULL; + d_stream.opaque = Z_NULL; + d_stream.avail_in = 0; + d_stream.next_in = Z_NULL; + err = inflateInit2(&d_stream, -MAX_WBITS); - d_stream.avail_in = compr_bytes; - d_stream.next_in = &buffer_compr[0]; - d_stream.avail_out = uncompr_bytes; - d_stream.next_out = &buffer_uncompr[0]; + d_stream.avail_in = compr_bytes; + d_stream.next_in = &buffer_compr[0]; + d_stream.avail_out = uncompr_bytes; + d_stream.next_out = &buffer_uncompr[0]; - err = inflate(&d_stream, Z_FINISH); - err = inflateEnd(&d_stream); + err = inflate(&d_stream, Z_FINISH); + err = inflateEnd(&d_stream); - std::vector shape; - size_t word_size; - bool fortran_order; - cnpy::parse_npy_header(&buffer_uncompr[0],word_size,shape,fortran_order); + std::vector shape; + size_t word_size; + bool fortran_order; + cnpy::parse_npy_header(&buffer_uncompr[0], word_size, shape, fortran_order); - cnpy::NpyArray array(shape, word_size, fortran_order); + cnpy::NpyArray array(shape, word_size, fortran_order); - size_t offset = uncompr_bytes - array.num_bytes(); - memcpy(array.data(),&buffer_uncompr[0]+offset,array.num_bytes()); + size_t offset = uncompr_bytes - array.num_bytes(); + memcpy(array.data(), &buffer_uncompr[0] + offset, + array.num_bytes()); - return array; + return array; } cnpy::npz_t cnpy::npz_load(std::string fname) { - FILE* fp = fopen(fname.c_str(),"rb"); - - if(!fp) { - throw std::runtime_error("npz_load: Error! Unable to open file "+fname+"!"); + FILE* fp = fopen(fname.c_str(), "rb"); + + if (!fp) { + throw std::runtime_error("npz_load: Error! Unable to open file " + fname + + "!"); + } + + cnpy::npz_t arrays; + + while (1) { + std::vector local_header(30); + size_t headerres = fread(&local_header[0], sizeof(char), 30, fp); + if (headerres != 30) + throw std::runtime_error("npz_load: failed fread"); + + // if we've reached the global header, stop reading + if (local_header[2] != 0x03 || local_header[3] != 0x04) + break; + + // read in the variable name + uint16_t name_len = *(uint16_t*) &local_header[26]; + std::string varname(name_len, ' '); + size_t vname_res = fread(&varname[0], sizeof(char), name_len, fp); + if (vname_res != name_len) + throw std::runtime_error("npz_load: failed fread"); + + // erase the lagging .npy + varname.erase(varname.end() - 4, varname.end()); + + // read in the extra field + uint16_t extra_field_len = *(uint16_t*) &local_header[28]; + if (extra_field_len > 0) { + std::vector buff(extra_field_len); + size_t efield_res = fread(&buff[0], sizeof(char), extra_field_len, fp); + if (efield_res != extra_field_len) + throw std::runtime_error("npz_load: failed fread"); } - cnpy::npz_t arrays; - - while(1) { - std::vector local_header(30); - size_t headerres = fread(&local_header[0],sizeof(char),30,fp); - if(headerres != 30) - throw std::runtime_error("npz_load: failed fread"); - - //if we've reached the global header, stop reading - if(local_header[2] != 0x03 || local_header[3] != 0x04) break; - - //read in the variable name - uint16_t name_len = *(uint16_t*) &local_header[26]; - std::string varname(name_len,' '); - size_t vname_res = fread(&varname[0],sizeof(char),name_len,fp); - if(vname_res != name_len) - throw std::runtime_error("npz_load: failed fread"); - - //erase the lagging .npy - varname.erase(varname.end()-4,varname.end()); - - //read in the extra field - uint16_t extra_field_len = *(uint16_t*) &local_header[28]; - if(extra_field_len > 0) { - std::vector buff(extra_field_len); - size_t efield_res = fread(&buff[0],sizeof(char),extra_field_len,fp); - if(efield_res != extra_field_len) - throw std::runtime_error("npz_load: failed fread"); - } - - uint16_t compr_method = *reinterpret_cast(&local_header[0]+8); - uint32_t compr_bytes = *reinterpret_cast(&local_header[0]+18); - uint32_t uncompr_bytes = *reinterpret_cast(&local_header[0]+22); - - if(compr_method == 0) {arrays[varname] = load_the_npy_file(fp);} - else {arrays[varname] = load_the_npz_array(fp,compr_bytes,uncompr_bytes);} + uint16_t compr_method = *reinterpret_cast(&local_header[0] + 8); + uint32_t compr_bytes = *reinterpret_cast(&local_header[0] + 18); + uint32_t uncompr_bytes = + *reinterpret_cast(&local_header[0] + 22); + + if (compr_method == 0) { + arrays[varname] = load_the_npy_file(fp); + } else { + arrays[varname] = load_the_npz_array(fp, compr_bytes, uncompr_bytes); } + } - fclose(fp); - return arrays; + fclose(fp); + return arrays; } cnpy::NpyArray cnpy::npz_load(std::string fname, std::string varname) { - FILE* fp = fopen(fname.c_str(),"rb"); - - if(!fp) throw std::runtime_error("npz_load: Unable to open file "+fname); - - while(1) { - std::vector local_header(30); - size_t header_res = fread(&local_header[0],sizeof(char),30,fp); - if(header_res != 30) - throw std::runtime_error("npz_load: failed fread"); - - //if we've reached the global header, stop reading - if(local_header[2] != 0x03 || local_header[3] != 0x04) break; - - //read in the variable name - uint16_t name_len = *(uint16_t*) &local_header[26]; - std::string vname(name_len,' '); - size_t vname_res = fread(&vname[0],sizeof(char),name_len,fp); - if(vname_res != name_len) - throw std::runtime_error("npz_load: failed fread"); - vname.erase(vname.end()-4,vname.end()); //erase the lagging .npy - - //read in the extra field - uint16_t extra_field_len = *(uint16_t*) &local_header[28]; - fseek(fp,extra_field_len,SEEK_CUR); //skip past the extra field - - uint16_t compr_method = *reinterpret_cast(&local_header[0]+8); - uint32_t compr_bytes = *reinterpret_cast(&local_header[0]+18); - uint32_t uncompr_bytes = *reinterpret_cast(&local_header[0]+22); - - if(vname == varname) { - NpyArray array = (compr_method == 0) ? load_the_npy_file(fp) : load_the_npz_array(fp,compr_bytes,uncompr_bytes); - fclose(fp); - return array; - } - else { - //skip past the data - uint32_t size = *(uint32_t*) &local_header[22]; - fseek(fp,size,SEEK_CUR); - } + FILE* fp = fopen(fname.c_str(), "rb"); + + if (!fp) + throw std::runtime_error("npz_load: Unable to open file " + fname); + + while (1) { + std::vector local_header(30); + size_t header_res = fread(&local_header[0], sizeof(char), 30, fp); + if (header_res != 30) + throw std::runtime_error("npz_load: failed fread"); + + // if we've reached the global header, stop reading + if (local_header[2] != 0x03 || local_header[3] != 0x04) + break; + + // read in the variable name + uint16_t name_len = *(uint16_t*) &local_header[26]; + std::string vname(name_len, ' '); + size_t vname_res = fread(&vname[0], sizeof(char), name_len, fp); + if (vname_res != name_len) + throw std::runtime_error("npz_load: failed fread"); + vname.erase(vname.end() - 4, vname.end()); // erase the lagging .npy + + // read in the extra field + uint16_t extra_field_len = *(uint16_t*) &local_header[28]; + fseek(fp, extra_field_len, SEEK_CUR); // skip past the extra field + + uint16_t compr_method = *reinterpret_cast(&local_header[0] + 8); + uint32_t compr_bytes = *reinterpret_cast(&local_header[0] + 18); + uint32_t uncompr_bytes = + *reinterpret_cast(&local_header[0] + 22); + + if (vname == varname) { + NpyArray array = (compr_method == 0) + ? load_the_npy_file(fp) + : load_the_npz_array(fp, compr_bytes, uncompr_bytes); + fclose(fp); + return array; + } else { + // skip past the data + uint32_t size = *(uint32_t*) &local_header[22]; + fseek(fp, size, SEEK_CUR); } + } - fclose(fp); + fclose(fp); - //if we get here, we haven't found the variable in the file - throw std::runtime_error("npz_load: Variable name "+varname+" not found in "+fname); + // if we get here, we haven't found the variable in the file + throw std::runtime_error("npz_load: Variable name " + varname + + " not found in " + fname); } cnpy::NpyArray cnpy::npy_load(std::string fname) { - FILE* fp = fopen(fname.c_str(), "rb"); + FILE* fp = fopen(fname.c_str(), "rb"); - if(!fp) throw std::runtime_error("npy_load: Unable to open file "+fname); + if (!fp) + throw std::runtime_error("npy_load: Unable to open file " + fname); - NpyArray arr = load_the_npy_file(fp); + NpyArray arr = load_the_npy_file(fp); - fclose(fp); - return arr; + fclose(fp); + return arr; } - - diff --git a/third_party/cnpy/cnpy.h b/third_party/cnpy/cnpy.h index 08bd303..9361e60 100644 --- a/third_party/cnpy/cnpy.h +++ b/third_party/cnpy/cnpy.h @@ -1,271 +1,310 @@ -//Copyright (C) 2011 Carl Rogers -//Released under MIT License -//license available in LICENSE file, or at http://www.opensource.org/licenses/mit-license.php +// SPDX-FileCopyrightText: 2011 Carl Rogers +// +// SPDX-License-Identifier: MIT + +// Copyright (C) 2011 Carl Rogers +// Released under MIT License +// license available in LICENSE file, or at +// http://www.opensource.org/licenses/mit-license.php #ifndef LIBCNPY_H_ #define LIBCNPY_H_ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include namespace cnpy { - struct NpyArray { - NpyArray(const std::vector& _shape, size_t _word_size, bool _fortran_order) : - shape(_shape), word_size(_word_size), fortran_order(_fortran_order) - { - num_vals = 1; - for(size_t i = 0;i < shape.size();i++) num_vals *= shape[i]; - data_holder = std::shared_ptr>( - new std::vector(num_vals * word_size)); - } - - NpyArray() : shape(0), word_size(0), fortran_order(0), num_vals(0) { } - - template - T* data() { - return reinterpret_cast(&(*data_holder)[0]); - } - - template - const T* data() const { - return reinterpret_cast(&(*data_holder)[0]); - } - - template - std::vector as_vec() const { - const T* p = data(); - return std::vector(p, p+num_vals); - } - - size_t num_bytes() const { - return data_holder->size(); - } - - std::shared_ptr> data_holder; - std::vector shape; - size_t word_size; - bool fortran_order; - size_t num_vals; - }; - - using npz_t = std::map; - - char BigEndianTest(); - char map_type(const std::type_info& t); - template std::vector create_npy_header(const std::vector& shape); - void parse_npy_header(FILE* fp,size_t& word_size, std::vector& shape, bool& fortran_order); - void parse_npy_header(unsigned char* buffer,size_t& word_size, std::vector& shape, bool& fortran_order); - void parse_zip_footer(FILE* fp, uint16_t& nrecs, size_t& global_header_size, size_t& global_header_offset); - npz_t npz_load(std::string fname); - NpyArray npz_load(std::string fname, std::string varname); - NpyArray npy_load(std::string fname); - - template std::vector& operator+=(std::vector& lhs, const T rhs) { - //write in little endian - for(size_t byte = 0; byte < sizeof(T); byte++) { - char val = *((char*)&rhs+byte); - lhs.push_back(val); - } - return lhs; - } +struct NpyArray { + NpyArray(const std::vector& _shape, size_t _word_size, + bool _fortran_order) + : shape(_shape), word_size(_word_size), fortran_order(_fortran_order) { + num_vals = 1; + for (size_t i = 0; i < shape.size(); i++) + num_vals *= shape[i]; + data_holder = std::shared_ptr>( + new std::vector(num_vals * word_size)); + } + + NpyArray() : shape(0), word_size(0), fortran_order(0), num_vals(0) {} + + template + T* data() { + return reinterpret_cast(&(*data_holder)[0]); + } + + template + const T* data() const { + return reinterpret_cast(&(*data_holder)[0]); + } + + template + std::vector as_vec() const { + const T* p = data(); + return std::vector(p, p + num_vals); + } + + size_t num_bytes() const { + return data_holder->size(); + } + + std::shared_ptr> data_holder; + std::vector shape; + size_t word_size; + bool fortran_order; + size_t num_vals; +}; + +using npz_t = std::map; + +char BigEndianTest(); +char map_type(const std::type_info& t); +template +std::vector create_npy_header(const std::vector& shape); +void parse_npy_header(FILE* fp, size_t& word_size, std::vector& shape, + bool& fortran_order); +void parse_npy_header(unsigned char* buffer, size_t& word_size, + std::vector& shape, bool& fortran_order); +void parse_zip_footer(FILE* fp, uint16_t& nrecs, size_t& global_header_size, + size_t& global_header_offset); +npz_t npz_load(std::string fname); +NpyArray npz_load(std::string fname, std::string varname); +NpyArray npy_load(std::string fname); + +template +std::vector& operator+=(std::vector& lhs, const T rhs) { + // write in little endian + for (size_t byte = 0; byte < sizeof(T); byte++) { + char val = *((char*) &rhs + byte); + lhs.push_back(val); + } + return lhs; +} - template<> std::vector& operator+=(std::vector& lhs, const std::string rhs); - template<> std::vector& operator+=(std::vector& lhs, const char* rhs); - - - template void npy_save(std::string fname, const T* data, const std::vector shape, std::string mode = "w") { - FILE* fp = NULL; - std::vector true_data_shape; //if appending, the shape of existing + new data - - if(mode == "a") fp = fopen(fname.c_str(),"r+b"); - - if(fp) { - //file exists. we need to append to it. read the header, modify the array size - size_t word_size; - bool fortran_order; - parse_npy_header(fp,word_size,true_data_shape,fortran_order); - assert(!fortran_order); - - if(word_size != sizeof(T)) { - std::cout<<"libnpy error: "< header = create_npy_header(true_data_shape); - size_t nels = std::accumulate(shape.begin(),shape.end(),1,std::multiplies()); - - fseek(fp,0,SEEK_SET); - fwrite(&header[0],sizeof(char),header.size(),fp); - fseek(fp,0,SEEK_END); - fwrite(data,sizeof(T),nels,fp); - fclose(fp); +template <> +std::vector& operator+=(std::vector& lhs, const std::string rhs); +template <> +std::vector& operator+=(std::vector& lhs, const char* rhs); + +template +void npy_save(std::string fname, const T* data, const std::vector shape, + std::string mode = "w") { + FILE* fp = NULL; + std::vector + true_data_shape; // if appending, the shape of existing + new data + + if (mode == "a") + fp = fopen(fname.c_str(), "r+b"); + + if (fp) { + // file exists. we need to append to it. read the header, modify the array + // size + size_t word_size; + bool fortran_order; + parse_npy_header(fp, word_size, true_data_shape, fortran_order); + assert(!fortran_order); + + if (word_size != sizeof(T)) { + std::cout << "libnpy error: " << fname << " has word size " << word_size + << " but npy_save appending data sized " << sizeof(T) << "\n"; + assert(word_size == sizeof(T)); } - - template void npz_save(std::string zipname, std::string fname, const T* data, const std::vector& shape, std::string mode = "w") - { - //first, append a .npy to the fname - fname += ".npy"; - - //now, on with the show - FILE* fp = NULL; - uint16_t nrecs = 0; - size_t global_header_offset = 0; - std::vector global_header; - - if(mode == "a") fp = fopen(zipname.c_str(),"r+b"); - - if(fp) { - //zip file exists. we need to add a new npy file to it. - //first read the footer. this gives us the offset and size of the global header - //then read and store the global header. - //below, we will write the the new data at the start of the global header then append the global header and footer below it - size_t global_header_size; - parse_zip_footer(fp,nrecs,global_header_size,global_header_offset); - fseek(fp,global_header_offset,SEEK_SET); - global_header.resize(global_header_size); - size_t res = fread(&global_header[0],sizeof(char),global_header_size,fp); - if(res != global_header_size){ - throw std::runtime_error("npz_save: header read error while adding to existing zip"); - } - fseek(fp,global_header_offset,SEEK_SET); - } - else { - fp = fopen(zipname.c_str(),"wb"); - } - - std::vector npy_header = create_npy_header(shape); - - size_t nels = std::accumulate(shape.begin(),shape.end(),1,std::multiplies()); - size_t nbytes = nels*sizeof(T) + npy_header.size(); - - //get the CRC of the data to be added - uint32_t crc = crc32(0L,(uint8_t*)&npy_header[0],npy_header.size()); - crc = crc32(crc,(uint8_t*)data,nels*sizeof(T)); - - //build the local header - std::vector local_header; - local_header += "PK"; //first part of sig - local_header += (uint16_t) 0x0403; //second part of sig - local_header += (uint16_t) 20; //min version to extract - local_header += (uint16_t) 0; //general purpose bit flag - local_header += (uint16_t) 0; //compression method - local_header += (uint16_t) 0; //file last mod time - local_header += (uint16_t) 0; //file last mod date - local_header += (uint32_t) crc; //crc - local_header += (uint32_t) nbytes; //compressed size - local_header += (uint32_t) nbytes; //uncompressed size - local_header += (uint16_t) fname.size(); //fname length - local_header += (uint16_t) 0; //extra field length - local_header += fname; - - //build global header - global_header += "PK"; //first part of sig - global_header += (uint16_t) 0x0201; //second part of sig - global_header += (uint16_t) 20; //version made by - global_header.insert(global_header.end(),local_header.begin()+4,local_header.begin()+30); - global_header += (uint16_t) 0; //file comment length - global_header += (uint16_t) 0; //disk number where file starts - global_header += (uint16_t) 0; //internal file attributes - global_header += (uint32_t) 0; //external file attributes - global_header += (uint32_t) global_header_offset; //relative offset of local file header, since it begins where the global header used to begin - global_header += fname; - - //build footer - std::vector footer; - footer += "PK"; //first part of sig - footer += (uint16_t) 0x0605; //second part of sig - footer += (uint16_t) 0; //number of this disk - footer += (uint16_t) 0; //disk where footer starts - footer += (uint16_t) (nrecs+1); //number of records on this disk - footer += (uint16_t) (nrecs+1); //total number of records - footer += (uint32_t) global_header.size(); //nbytes of global headers - footer += (uint32_t) (global_header_offset + nbytes + local_header.size()); //offset of start of global headers, since global header now starts after newly written array - footer += (uint16_t) 0; //zip file comment length - - //write everything - fwrite(&local_header[0],sizeof(char),local_header.size(),fp); - fwrite(&npy_header[0],sizeof(char),npy_header.size(),fp); - fwrite(data,sizeof(T),nels,fp); - fwrite(&global_header[0],sizeof(char),global_header.size(),fp); - fwrite(&footer[0],sizeof(char),footer.size(),fp); - fclose(fp); + if (true_data_shape.size() != shape.size()) { + std::cout << "libnpy error: npy_save attempting to append misdimensioned " + "data to " + << fname << "\n"; + assert(true_data_shape.size() != shape.size()); } - template void npy_save(std::string fname, const std::vector data, std::string mode = "w") { - std::vector shape; - shape.push_back(data.size()); - npy_save(fname, &data[0], shape, mode); + for (size_t i = 1; i < shape.size(); i++) { + if (shape[i] != true_data_shape[i]) { + std::cout + << "libnpy error: npy_save attempting to append misshaped data to " + << fname << "\n"; + assert(shape[i] == true_data_shape[i]); + } } + true_data_shape[0] += shape[0]; + } else { + fp = fopen(fname.c_str(), "wb"); + true_data_shape = shape; + } + + std::vector header = create_npy_header(true_data_shape); + size_t nels = + std::accumulate(shape.begin(), shape.end(), 1, std::multiplies()); + + fseek(fp, 0, SEEK_SET); + fwrite(&header[0], sizeof(char), header.size(), fp); + fseek(fp, 0, SEEK_END); + fwrite(data, sizeof(T), nels, fp); + fclose(fp); +} - template void npz_save(std::string zipname, std::string fname, const std::vector data, std::string mode = "w") { - std::vector shape; - shape.push_back(data.size()); - npz_save(zipname, fname, &data[0], shape, mode); +template +void npz_save(std::string zipname, std::string fname, const T* data, + const std::vector& shape, std::string mode = "w") { + // first, append a .npy to the fname + fname += ".npy"; + + // now, on with the show + FILE* fp = NULL; + uint16_t nrecs = 0; + size_t global_header_offset = 0; + std::vector global_header; + + if (mode == "a") + fp = fopen(zipname.c_str(), "r+b"); + + if (fp) { + // zip file exists. we need to add a new npy file to it. + // first read the footer. this gives us the offset and size of the global + // header then read and store the global header. below, we will write the + // the new data at the start of the global header then append the global + // header and footer below it + size_t global_header_size; + parse_zip_footer(fp, nrecs, global_header_size, global_header_offset); + fseek(fp, global_header_offset, SEEK_SET); + global_header.resize(global_header_size); + size_t res = fread(&global_header[0], sizeof(char), global_header_size, fp); + if (res != global_header_size) { + throw std::runtime_error( + "npz_save: header read error while adding to existing zip"); } + fseek(fp, global_header_offset, SEEK_SET); + } else { + fp = fopen(zipname.c_str(), "wb"); + } + + std::vector npy_header = create_npy_header(shape); + + size_t nels = + std::accumulate(shape.begin(), shape.end(), 1, std::multiplies()); + size_t nbytes = nels * sizeof(T) + npy_header.size(); + + // get the CRC of the data to be added + uint32_t crc = crc32(0L, (uint8_t*) &npy_header[0], npy_header.size()); + crc = crc32(crc, (uint8_t*) data, nels * sizeof(T)); + + // build the local header + std::vector local_header; + local_header += "PK"; // first part of sig + local_header += (uint16_t) 0x0403; // second part of sig + local_header += (uint16_t) 20; // min version to extract + local_header += (uint16_t) 0; // general purpose bit flag + local_header += (uint16_t) 0; // compression method + local_header += (uint16_t) 0; // file last mod time + local_header += (uint16_t) 0; // file last mod date + local_header += (uint32_t) crc; // crc + local_header += (uint32_t) nbytes; // compressed size + local_header += (uint32_t) nbytes; // uncompressed size + local_header += (uint16_t) fname.size(); // fname length + local_header += (uint16_t) 0; // extra field length + local_header += fname; + + // build global header + global_header += "PK"; // first part of sig + global_header += (uint16_t) 0x0201; // second part of sig + global_header += (uint16_t) 20; // version made by + global_header.insert(global_header.end(), local_header.begin() + 4, + local_header.begin() + 30); + global_header += (uint16_t) 0; // file comment length + global_header += (uint16_t) 0; // disk number where file starts + global_header += (uint16_t) 0; // internal file attributes + global_header += (uint32_t) 0; // external file attributes + global_header += (uint32_t) + global_header_offset; // relative offset of local file header, since it + // begins where the global header used to begin + global_header += fname; + + // build footer + std::vector footer; + footer += "PK"; // first part of sig + footer += (uint16_t) 0x0605; // second part of sig + footer += (uint16_t) 0; // number of this disk + footer += (uint16_t) 0; // disk where footer starts + footer += (uint16_t) (nrecs + 1); // number of records on this disk + footer += (uint16_t) (nrecs + 1); // total number of records + footer += (uint32_t) global_header.size(); // nbytes of global headers + footer += + (uint32_t) (global_header_offset + nbytes + + local_header.size()); // offset of start of global headers, + // since global header now starts after + // newly written array + footer += (uint16_t) 0; // zip file comment length + + // write everything + fwrite(&local_header[0], sizeof(char), local_header.size(), fp); + fwrite(&npy_header[0], sizeof(char), npy_header.size(), fp); + fwrite(data, sizeof(T), nels, fp); + fwrite(&global_header[0], sizeof(char), global_header.size(), fp); + fwrite(&footer[0], sizeof(char), footer.size(), fp); + fclose(fp); +} - template std::vector create_npy_header(const std::vector& shape) { - - std::vector dict; - dict += "{'descr': '"; - dict += BigEndianTest(); - dict += map_type(typeid(T)); - dict += std::to_string(sizeof(T)); - dict += "', 'fortran_order': False, 'shape': ("; - if(!shape.empty()) { - dict += std::to_string(shape[0]); - for(size_t i = 1;i < shape.size();i++) { - dict += ", "; - dict += std::to_string(shape[i]); - } - } - if(shape.size() == 1) dict += ","; - dict += "), }"; - //pad with spaces so that preamble+dict is modulo 16 bytes. preamble is 10 bytes. dict needs to end with \n - int remainder = 16 - (10 + dict.size()) % 16; - dict.insert(dict.end(),remainder,' '); - dict.back() = '\n'; - - std::vector header; - header += (char) 0x93; - header += "NUMPY"; - header += (char) 0x01; //major version of numpy format - header += (char) 0x00; //minor version of numpy format - header += (uint16_t) dict.size(); - header.insert(header.end(),dict.begin(),dict.end()); - - return header; - } +template +void npy_save(std::string fname, const std::vector data, + std::string mode = "w") { + std::vector shape; + shape.push_back(data.size()); + npy_save(fname, &data[0], shape, mode); +} +template +void npz_save(std::string zipname, std::string fname, const std::vector data, + std::string mode = "w") { + std::vector shape; + shape.push_back(data.size()); + npz_save(zipname, fname, &data[0], shape, mode); +} +template +std::vector create_npy_header(const std::vector& shape) { + + std::vector dict; + dict += "{'descr': '"; + dict += BigEndianTest(); + dict += map_type(typeid(T)); + dict += std::to_string(sizeof(T)); + dict += "', 'fortran_order': False, 'shape': ("; + if (!shape.empty()) { + dict += std::to_string(shape[0]); + for (size_t i = 1; i < shape.size(); i++) { + dict += ", "; + dict += std::to_string(shape[i]); + } + } + if (shape.size() == 1) + dict += ","; + dict += "), }"; + // pad with spaces so that preamble+dict is modulo 16 bytes. preamble is 10 + // bytes. dict needs to end with \n + int remainder = 16 - (10 + dict.size()) % 16; + dict.insert(dict.end(), remainder, ' '); + dict.back() = '\n'; + + std::vector header; + header += (char) 0x93; + header += "NUMPY"; + header += (char) 0x01; // major version of numpy format + header += (char) 0x00; // minor version of numpy format + header += (uint16_t) dict.size(); + header.insert(header.end(), dict.begin(), dict.end()); + + return header; } +} // namespace cnpy + #endif diff --git a/third_party/cnpy/example1.cpp b/third_party/cnpy/example1.cpp index 70ac5aa..3ea5536 100644 --- a/third_party/cnpy/example1.cpp +++ b/third_party/cnpy/example1.cpp @@ -1,55 +1,64 @@ -#include"cnpy.h" -#include -#include -#include -#include -#include +// SPDX-FileCopyrightText: 2011 Carl Rogers +// +// SPDX-License-Identifier: MIT + +#include "cnpy.h" +#include +#include +#include +#include +#include const int Nx = 128; const int Ny = 64; const int Nz = 32; -int main() -{ - //set random seed so that result is reproducible (for testing) - srand(0); - //create random data - std::vector> data(Nx*Ny*Nz); - for(int i = 0;i < Nx*Ny*Nz;i++) data[i] = std::complex(rand(),rand()); - - //save it to file - cnpy::npy_save("arr1.npy",&data[0],{Nz,Ny,Nx},"w"); - - //load it into a new array - cnpy::NpyArray arr = cnpy::npy_load("arr1.npy"); - std::complex* loaded_data = arr.data>(); - - //make sure the loaded data matches the saved data - assert(arr.word_size == sizeof(std::complex)); - assert(arr.shape.size() == 3 && arr.shape[0] == Nz && arr.shape[1] == Ny && arr.shape[2] == Nx); - for(int i = 0; i < Nx*Ny*Nz;i++) assert(data[i] == loaded_data[i]); - - //append the same data to file - //npy array on file now has shape (Nz+Nz,Ny,Nx) - cnpy::npy_save("arr1.npy",&data[0],{Nz,Ny,Nx},"a"); - - //now write to an npz file - //non-array variables are treated as 1D arrays with 1 element - double myVar1 = 1.2; - char myVar2 = 'a'; - cnpy::npz_save("out.npz","myVar1",&myVar1,{1},"w"); //"w" overwrites any existing file - cnpy::npz_save("out.npz","myVar2",&myVar2,{1},"a"); //"a" appends to the file we created above - cnpy::npz_save("out.npz","arr1",&data[0],{Nz,Ny,Nx},"a"); //"a" appends to the file we created above - - //load a single var from the npz file - cnpy::NpyArray arr2 = cnpy::npz_load("out.npz","arr1"); - - //load the entire npz file - cnpy::npz_t my_npz = cnpy::npz_load("out.npz"); - - //check that the loaded myVar1 matches myVar1 - cnpy::NpyArray arr_mv1 = my_npz["myVar1"]; - double* mv1 = arr_mv1.data(); - assert(arr_mv1.shape.size() == 1 && arr_mv1.shape[0] == 1); - assert(mv1[0] == myVar1); +int main() { + // set random seed so that result is reproducible (for testing) + srand(0); + // create random data + std::vector> data(Nx * Ny * Nz); + for (int i = 0; i < Nx * Ny * Nz; i++) + data[i] = std::complex(rand(), rand()); + + // save it to file + cnpy::npy_save("arr1.npy", &data[0], {Nz, Ny, Nx}, "w"); + + // load it into a new array + cnpy::NpyArray arr = cnpy::npy_load("arr1.npy"); + std::complex* loaded_data = arr.data>(); + + // make sure the loaded data matches the saved data + assert(arr.word_size == sizeof(std::complex)); + assert(arr.shape.size() == 3 && arr.shape[0] == Nz && arr.shape[1] == Ny && + arr.shape[2] == Nx); + for (int i = 0; i < Nx * Ny * Nz; i++) + assert(data[i] == loaded_data[i]); + + // append the same data to file + // npy array on file now has shape (Nz+Nz,Ny,Nx) + cnpy::npy_save("arr1.npy", &data[0], {Nz, Ny, Nx}, "a"); + + // now write to an npz file + // non-array variables are treated as 1D arrays with 1 element + double myVar1 = 1.2; + char myVar2 = 'a'; + cnpy::npz_save("out.npz", "myVar1", &myVar1, {1}, + "w"); //"w" overwrites any existing file + cnpy::npz_save("out.npz", "myVar2", &myVar2, {1}, + "a"); //"a" appends to the file we created above + cnpy::npz_save("out.npz", "arr1", &data[0], {Nz, Ny, Nx}, + "a"); //"a" appends to the file we created above + + // load a single var from the npz file + cnpy::NpyArray arr2 = cnpy::npz_load("out.npz", "arr1"); + + // load the entire npz file + cnpy::npz_t my_npz = cnpy::npz_load("out.npz"); + + // check that the loaded myVar1 matches myVar1 + cnpy::NpyArray arr_mv1 = my_npz["myVar1"]; + double* mv1 = arr_mv1.data(); + assert(arr_mv1.shape.size() == 1 && arr_mv1.shape[0] == 1); + assert(mv1[0] == myVar1); } diff --git a/third_party/cnpy/mat2npz b/third_party/cnpy/mat2npz index c1bbd3e..6259342 100644 --- a/third_party/cnpy/mat2npz +++ b/third_party/cnpy/mat2npz @@ -1,5 +1,9 @@ #!/usr/bin/env python +# SPDX-FileCopyrightText: 2011 Carl Rogers +# +# SPDX-License-Identifier: MIT + import sys from numpy import savez from scipy.io import loadmat diff --git a/third_party/cnpy/npy2mat b/third_party/cnpy/npy2mat index fdd5da7..1eeecc7 100644 --- a/third_party/cnpy/npy2mat +++ b/third_party/cnpy/npy2mat @@ -1,5 +1,9 @@ #!/usr/bin/env python +# SPDX-FileCopyrightText: 2011 Carl Rogers +# +# SPDX-License-Identifier: MIT + import sys from numpy import load from scipy.io import savemat diff --git a/third_party/cnpy/npz2mat b/third_party/cnpy/npz2mat index 2e317d6..14e89e6 100755 --- a/third_party/cnpy/npz2mat +++ b/third_party/cnpy/npz2mat @@ -1,5 +1,9 @@ #!/usr/bin/env python +# SPDX-FileCopyrightText: 2011 Carl Rogers +# +# SPDX-License-Identifier: MIT + import sys from numpy import load from scipy.io import savemat From dda39a4d4686a2e262be9ae6dcddfc90905aad69 Mon Sep 17 00:00:00 2001 From: Willow Ahrens Date: Fri, 28 Aug 2026 12:03:55 -0400 Subject: [PATCH 08/16] compliance testing --- include/binsparse/array.h | 7 ++-- include/binsparse/hdf5_wrapper.h | 43 +++++++++++++----------- src/write_matrix.c | 12 +++---- test/compliance/binsparse_to_binsparse.c | 7 ++++ 4 files changed, 40 insertions(+), 29 deletions(-) diff --git a/include/binsparse/array.h b/include/binsparse/array.h index 0570d10..e27b1af 100644 --- a/include/binsparse/array.h +++ b/include/binsparse/array.h @@ -40,9 +40,9 @@ bsp_construct_array_t_allocator(bsp_array_t* array, size_t size, size_t byte_size = size * bsp_type_size(type); array->allocator = allocator; - array->data = array->allocator.malloc(byte_size); + array->data = byte_size == 0 ? NULL : array->allocator.malloc(byte_size); - if (array->data == NULL) { + if (byte_size != 0 && array->data == NULL) { return BSP_ERROR_MEMORY; } @@ -65,7 +65,8 @@ static inline bsp_error_t bsp_copy_construct_array_t(bsp_array_t* array, return error; } - memcpy(array->data, other.data, other.size * bsp_type_size(other.type)); + if (other.size != 0) + memcpy(array->data, other.data, other.size * bsp_type_size(other.type)); return BSP_SUCCESS; } diff --git a/include/binsparse/hdf5_wrapper.h b/include/binsparse/hdf5_wrapper.h index b52423c..833be10 100644 --- a/include/binsparse/hdf5_wrapper.h +++ b/include/binsparse/hdf5_wrapper.h @@ -42,19 +42,21 @@ static inline bsp_error_t bsp_write_array(hid_t f, const char* label, hid_t dcpl = H5Pcreate(H5P_DATASET_CREATE); - // Choose 1 MiB, the default chunk cache size, as our chunk size. - size_t chunk_size = 1024 * 1024 / bsp_type_size(array.type); + if (array.size > 0) { + // Choose 1 MiB, the default chunk cache size, as our chunk size. + size_t chunk_size = 1024 * 1024 / bsp_type_size(array.type); - // If the dataset is smaller than the chunk size, cap the chunk size. - if (array.size < chunk_size) { - chunk_size = array.size; - } + // If the dataset is smaller than the chunk size, cap the chunk size. + if (array.size < chunk_size) { + chunk_size = array.size; + } - hsize[0] = chunk_size; - H5Pset_chunk(dcpl, 1, hsize); + hsize[0] = chunk_size; + H5Pset_chunk(dcpl, 1, hsize); - if (compression_level > 0) { - H5Pset_deflate(dcpl, compression_level); + if (compression_level > 0) { + H5Pset_deflate(dcpl, compression_level); + } } hid_t dset = @@ -67,17 +69,18 @@ static inline bsp_error_t bsp_write_array(hid_t f, const char* label, return BSP_ERROR_IO; } - hid_t hdf5_native_type = bsp_get_hdf5_native_type(array.type); + if (array.size > 0) { + hid_t hdf5_native_type = bsp_get_hdf5_native_type(array.type); + hid_t r = H5Dwrite(dset, hdf5_native_type, H5S_ALL, fspace, H5P_DEFAULT, + array.data); - hid_t r = H5Dwrite(dset, hdf5_native_type, H5S_ALL, fspace, H5P_DEFAULT, - array.data); - - if (r < 0) { - H5Dclose(dset); - H5Sclose(fspace); - H5Pclose(lcpl); - H5Pclose(dcpl); - return BSP_ERROR_IO; + if (r < 0) { + H5Dclose(dset); + H5Sclose(fspace); + H5Pclose(lcpl); + H5Pclose(dcpl); + return BSP_ERROR_IO; + } } H5Sclose(fspace); diff --git a/src/write_matrix.c b/src/write_matrix.c index fa1a26a..a7c09a1 100644 --- a/src/write_matrix.c +++ b/src/write_matrix.c @@ -69,17 +69,17 @@ char* bsp_generate_json(bsp_matrix_t matrix, cJSON* user_json) { free(type_string); } - if (matrix.indices_0.data != NULL) { + if (matrix.indices_0.type != BSP_INVALID_TYPE) { cJSON_AddStringToObject(data_types, "indices_0", bsp_get_type_string(matrix.indices_0.type)); } - if (matrix.indices_1.data != NULL) { + if (matrix.indices_1.type != BSP_INVALID_TYPE) { cJSON_AddStringToObject(data_types, "indices_1", bsp_get_type_string(matrix.indices_1.type)); } - if (matrix.pointers_to_1.data != NULL) { + if (matrix.pointers_to_1.type != BSP_INVALID_TYPE) { cJSON_AddStringToObject(data_types, "pointers_to_1", bsp_get_type_string(matrix.pointers_to_1.type)); } @@ -112,7 +112,7 @@ bsp_error_t bsp_write_matrix_to_group_cjson(hid_t f, bsp_matrix_t matrix, return error; } - if (matrix.indices_0.size > 0) { + if (matrix.indices_0.type != BSP_INVALID_TYPE) { error = bsp_write_array(f, (char*) "indices_0", matrix.indices_0, compression_level); if (error != BSP_SUCCESS) { @@ -120,7 +120,7 @@ bsp_error_t bsp_write_matrix_to_group_cjson(hid_t f, bsp_matrix_t matrix, } } - if (matrix.indices_1.size > 0) { + if (matrix.indices_1.type != BSP_INVALID_TYPE) { error = bsp_write_array(f, (char*) "indices_1", matrix.indices_1, compression_level); if (error != BSP_SUCCESS) { @@ -128,7 +128,7 @@ bsp_error_t bsp_write_matrix_to_group_cjson(hid_t f, bsp_matrix_t matrix, } } - if (matrix.pointers_to_1.size > 0) { + if (matrix.pointers_to_1.type != BSP_INVALID_TYPE) { error = bsp_write_array(f, (char*) "pointers_to_1", matrix.pointers_to_1, compression_level); if (error != BSP_SUCCESS) { diff --git a/test/compliance/binsparse_to_binsparse.c b/test/compliance/binsparse_to_binsparse.c index e833fc9..baf79e7 100644 --- a/test/compliance/binsparse_to_binsparse.c +++ b/test/compliance/binsparse_to_binsparse.c @@ -7,6 +7,7 @@ #include "reformat.h" #include +#include int main(int argc, char** argv) { cJSON* header; @@ -17,6 +18,12 @@ int main(int argc, char** argv) { return 2; } header = input_header(argv[1]); + cJSON* format = cJSON_GetObjectItemCaseSensitive(header, "format"); + const char* name = cJSON_GetStringValue(format); + if (name != NULL && strcmp(name, "DMAT") == 0) + cJSON_SetValuestring(format, "DMATR"); + else if (name != NULL && strcmp(name, "COO") == 0) + cJSON_SetValuestring(format, "COOR"); error = bsp_reformat_file(argv[1], argv[2], header, 0); cJSON_Delete(header); if (error != BSP_SUCCESS) { From 7019c564918b42e2796b52a9c071f3c8f836c3f6 Mon Sep 17 00:00:00 2001 From: Willow Ahrens Date: Sat, 29 Aug 2026 11:10:58 -0400 Subject: [PATCH 09/16] Fix compliance roundtrip edge cases --- include/binsparse/array.h | 4 +++- include/binsparse/hdf5_wrapper.h | 17 +++++++++++++++++ test/compliance/binsparse_to_binsparse.c | 8 +------- test/compliance/reformat.c | 15 +++++++++++++++ test/compliance/reformat.h | 1 + 5 files changed, 37 insertions(+), 8 deletions(-) diff --git a/include/binsparse/array.h b/include/binsparse/array.h index e27b1af..02b063c 100644 --- a/include/binsparse/array.h +++ b/include/binsparse/array.h @@ -105,7 +105,9 @@ static inline bsp_error_t bsp_fp_array_to_complex(bsp_array_t* array) { } static inline void bsp_destroy_array_t(bsp_array_t* array) { - array->allocator.free(array->data); + if (array->data != NULL) { + array->allocator.free(array->data); + } } static inline bool bsp_array_equal(bsp_array_t x, bsp_array_t y) { diff --git a/include/binsparse/hdf5_wrapper.h b/include/binsparse/hdf5_wrapper.h index 833be10..2d96f77 100644 --- a/include/binsparse/hdf5_wrapper.h +++ b/include/binsparse/hdf5_wrapper.h @@ -126,6 +126,16 @@ static inline bsp_error_t bsp_read_array_parallel(bsp_array_t* array, hid_t f, hid_t hdf5_type = H5Dget_type(dset); bsp_type_t type = bsp_get_bsp_type(hdf5_type); + if (dims[0] == 0) { + array->type = type; + array->size = 0; + array->data = NULL; + array->allocator = bsp_shm_allocator; + H5Tclose(hdf5_type); + H5Sclose(fspace); + H5Dclose(dset); + return BSP_SUCCESS; + } // Array will be written into a POSIX shared memory. bsp_shm_t array_shm = bsp_shm_new(dims[0] * bsp_type_size(type)); @@ -240,6 +250,13 @@ static inline bsp_error_t bsp_read_array_allocator(bsp_array_t* array, hid_t f, return BSP_ERROR_MEMORY; } + if (array->size == 0) { + H5Tclose(hdf5_type); + H5Sclose(fspace); + H5Dclose(dset); + return BSP_SUCCESS; + } + hid_t native_type = bsp_get_hdf5_native_type(type); herr_t status = H5Dread(dset, native_type, H5S_ALL, H5S_ALL, H5P_DEFAULT, array->data); diff --git a/test/compliance/binsparse_to_binsparse.c b/test/compliance/binsparse_to_binsparse.c index baf79e7..5dde053 100644 --- a/test/compliance/binsparse_to_binsparse.c +++ b/test/compliance/binsparse_to_binsparse.c @@ -7,7 +7,6 @@ #include "reformat.h" #include -#include int main(int argc, char** argv) { cJSON* header; @@ -18,12 +17,7 @@ int main(int argc, char** argv) { return 2; } header = input_header(argv[1]); - cJSON* format = cJSON_GetObjectItemCaseSensitive(header, "format"); - const char* name = cJSON_GetStringValue(format); - if (name != NULL && strcmp(name, "DMAT") == 0) - cJSON_SetValuestring(format, "DMATR"); - else if (name != NULL && strcmp(name, "COO") == 0) - cJSON_SetValuestring(format, "COOR"); + canonicalize_predefined_alias(header); error = bsp_reformat_file(argv[1], argv[2], header, 0); cJSON_Delete(header); if (error != BSP_SUCCESS) { diff --git a/test/compliance/reformat.c b/test/compliance/reformat.c index 48c8375..f3cf184 100644 --- a/test/compliance/reformat.c +++ b/test/compliance/reformat.c @@ -114,6 +114,21 @@ bool header_has_fill(cJSON* header) { return cJSON_IsTrue(cJSON_GetObjectItemCaseSensitive(header, "fill")); } +void canonicalize_predefined_alias(cJSON* header) { + cJSON* item = cJSON_GetObjectItemCaseSensitive(header, "format"); + const char* format = cJSON_GetStringValue(item); + const char* canonical = NULL; + if (format == NULL) + return; + if (!strcmp(format, "DMAT")) + canonical = "DMATR"; + else if (!strcmp(format, "COO")) + canonical = "COOR"; + if (canonical != NULL) + cJSON_ReplaceItemInObjectCaseSensitive(header, "format", + cJSON_CreateString(canonical)); +} + static bsp_error_t read_fill_value(const char* path, bsp_array_t* value) { hid_t file = H5Fopen(path, H5F_ACC_RDONLY, H5P_DEFAULT); bsp_error_t error; diff --git a/test/compliance/reformat.h b/test/compliance/reformat.h index 9348af2..88cad4e 100644 --- a/test/compliance/reformat.h +++ b/test/compliance/reformat.h @@ -37,6 +37,7 @@ cJSON* read_json(const char* path); cJSON* input_header(const char* path); bool header_values_are_iso(cJSON* header); bool header_has_fill(cJSON* header); +void canonicalize_predefined_alias(cJSON* header); void complete_output(const char* path, cJSON* requested, const bsp_array_t* fill_value); From e8d787bd64e0393c203da7f73bbdbb92b2ec642f Mon Sep 17 00:00:00 2001 From: Willow Ahrens Date: Sat, 29 Aug 2026 19:55:56 -0400 Subject: [PATCH 10/16] semver --- CMakeLists.txt | 12 +++++ examples/bsp-ls.c | 9 +++- include/binsparse/binsparse.h | 3 +- include/binsparse/hdf5_wrapper.h | 7 +-- include/binsparse/version.h | 21 ++++++++ src/CMakeLists.txt | 2 + src/read_matrix.c | 15 +++++- src/read_tensor.c | 8 +++ src/version.cpp | 47 +++++++++++++++++ test/c/CMakeLists.txt | 7 +++ test/c/dvec_shape_test.c | 2 +- test/c/version_test.c | 88 ++++++++++++++++++++++++++++++++ 12 files changed, 210 insertions(+), 11 deletions(-) create mode 100644 include/binsparse/version.h create mode 100644 src/version.cpp create mode 100644 test/c/version_test.c diff --git a/CMakeLists.txt b/CMakeLists.txt index 0106346..e997ee4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -48,12 +48,24 @@ FetchContent_Declare( ) FetchContent_MakeAvailable(cJSON) +# cpp-semver is header-only. Use strict SemVer parsing for file-format metadata. +set(SEMVER_BUILD_TESTS OFF CACHE INTERNAL "Disable cpp-semver tests") +set(SEMVER_INSTALL OFF CACHE INTERNAL "Disable cpp-semver install") +FetchContent_Declare( + cpp-semver + GIT_REPOSITORY https://github.com/z4kn4fein/cpp-semver.git + GIT_TAG v0.4.0 +) +FetchContent_MakeAvailable(cpp-semver) + # Restore original BUILD_SHARED_LIBS setting set(BUILD_SHARED_LIBS ${BUILD_SHARED_LIBS_BACKUP}) configure_file(${cJSON_SOURCE_DIR}/cJSON.h ${CMAKE_BINARY_DIR}/include/cJSON/cJSON.h) target_link_libraries(binsparse PRIVATE cjson) target_link_libraries(binsparse_dynamic PRIVATE cjson) +target_link_libraries(binsparse PRIVATE $) +target_link_libraries(binsparse_dynamic PRIVATE $) # Set up include directories properly for both build and install target_include_directories(${PROJECT_NAME} diff --git a/examples/bsp-ls.c b/examples/bsp-ls.c index 8c30e5d..a6807db 100644 --- a/examples/bsp-ls.c +++ b/examples/bsp-ls.c @@ -41,7 +41,14 @@ void print_group_info(hid_t g, const char* name) { assert(cJSON_IsString(version_)); char* version_string = cJSON_GetStringValue(version_); - // TODO: check version. + error = bsp_check_version_compatible(version_string); + if (error != BSP_SUCCESS) { + printf("Unsupported Binsparse version %s: %s\n", version_string, + bsp_get_error_string(error)); + cJSON_Delete(j); + free(json_string); + return; + } cJSON* format_ = cJSON_GetObjectItemCaseSensitive(binsparse, "format"); assert(format_ != NULL); diff --git a/include/binsparse/binsparse.h b/include/binsparse/binsparse.h index 189aa7c..fbb5f05 100644 --- a/include/binsparse/binsparse.h +++ b/include/binsparse/binsparse.h @@ -6,8 +6,6 @@ #pragma once -#define BINSPARSE_VERSION "0.1" - #include #include #include @@ -19,4 +17,5 @@ #include #include #include +#include #include diff --git a/include/binsparse/hdf5_wrapper.h b/include/binsparse/hdf5_wrapper.h index 2d96f77..bf331ef 100644 --- a/include/binsparse/hdf5_wrapper.h +++ b/include/binsparse/hdf5_wrapper.h @@ -142,6 +142,8 @@ static inline bsp_error_t bsp_read_array_parallel(bsp_array_t* array, hid_t f, array->type = type; array->size = dims[0]; array->allocator = bsp_shm_allocator; + array->data = bsp_shm_attach(array_shm); + bsp_shm_delete(array_shm); bsp_shm_t active_children_shm = bsp_shm_new(sizeof(_Atomic int)); @@ -165,11 +167,6 @@ static inline bsp_error_t bsp_read_array_parallel(bsp_array_t* array, hid_t f, } } - array->data = bsp_shm_attach(array_shm); - if (thread_num == 0) { - bsp_shm_delete(array_shm); - } - hsize_t chunk_size = (array->size + num_threads - 1) / num_threads; hsize_t start = (chunk_size * thread_num < array->size) ? chunk_size * thread_num diff --git a/include/binsparse/version.h b/include/binsparse/version.h new file mode 100644 index 0000000..008a65f --- /dev/null +++ b/include/binsparse/version.h @@ -0,0 +1,21 @@ +/* + * SPDX-FileCopyrightText: 2026 Binsparse Developers + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#pragma once + +#include + +#define BINSPARSE_VERSION "0.1.0" + +#ifdef __cplusplus +extern "C" { +#endif + +bsp_error_t bsp_check_version_compatible(const char* version); + +#ifdef __cplusplus +} +#endif diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index f966494..1afd501 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -5,6 +5,7 @@ target_sources(binsparse PRIVATE read_matrix.c read_tensor.c + version.cpp write_matrix.c write_tensor.c ) @@ -12,6 +13,7 @@ target_sources(binsparse PRIVATE target_sources(binsparse_dynamic PRIVATE read_matrix.c read_tensor.c + version.cpp write_matrix.c write_tensor.c ) diff --git a/src/read_matrix.c b/src/read_matrix.c index 4671f28..09b96c3 100644 --- a/src/read_matrix.c +++ b/src/read_matrix.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -91,7 +92,12 @@ bsp_error_t bsp_read_matrix_from_group_parallel(bsp_matrix_t* matrix, hid_t f, assert(cJSON_IsString(version_)); - // TODO: check version. + error = bsp_check_version_compatible(cJSON_GetStringValue(version_)); + if (error != BSP_SUCCESS) { + cJSON_Delete(j); + free(json_string); + return error; + } cJSON* format_ = cJSON_GetObjectItemCaseSensitive(binsparse, "format"); assert(format_ != NULL); @@ -237,7 +243,12 @@ bsp_error_t bsp_read_matrix_from_group_allocator(bsp_matrix_t* matrix, hid_t f, return BSP_ERROR_FORMAT; } - // TODO: check version. + error = bsp_check_version_compatible(cJSON_GetStringValue(version_)); + if (error != BSP_SUCCESS) { + cJSON_Delete(j); + allocator.free(json_string); + return error; + } cJSON* format_ = cJSON_GetObjectItemCaseSensitive(binsparse, "format"); if (format_ == NULL || !cJSON_IsString(format_)) { diff --git a/src/read_tensor.c b/src/read_tensor.c index 939c94f..ff1bc98 100644 --- a/src/read_tensor.c +++ b/src/read_tensor.c @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -52,6 +53,13 @@ bsp_tensor_t bsp_read_tensor_from_group(hid_t f) { assert(cJSON_IsString(version_)); + error = bsp_check_version_compatible(cJSON_GetStringValue(version_)); + if (error != BSP_SUCCESS) { + cJSON_Delete(j); + free(json_string); + return tensor; + } + // nnz computation cJSON* nnz_ = cJSON_GetObjectItemCaseSensitive(binsparse, "number_of_stored_values"); diff --git a/src/version.cpp b/src/version.cpp new file mode 100644 index 0000000..71595e0 --- /dev/null +++ b/src/version.cpp @@ -0,0 +1,47 @@ +/* + * SPDX-FileCopyrightText: 2026 Binsparse Developers + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include +#include + +#include + +namespace { + +bool bsp_semver_compatible(const semver::version& version, + const semver::version& supported) { + // Mirror python-semver Version.is_compatible(). + if (supported.major() == 0 && version.major() == 0 && + (supported.minor() != version.minor() || + supported.patch() != version.patch() || + supported.prerelease() != version.prerelease())) { + return false; + } + + return supported.major() == version.major() && + version.minor() >= supported.minor() && + supported.prerelease() == version.prerelease(); +} + +} // namespace + +extern "C" bsp_error_t bsp_check_version_compatible(const char* version) { + if (version == nullptr) { + return BSP_ERROR_FORMAT; + } + + try { + semver::version parsed = semver::version::parse(version); + semver::version supported = semver::version::parse(BINSPARSE_VERSION); + if (!bsp_semver_compatible(parsed, supported)) { + return BSP_ERROR_UNSUPPORTED; + } + } catch (const std::exception&) { + return BSP_ERROR_FORMAT; + } + + return BSP_SUCCESS; +} diff --git a/test/c/CMakeLists.txt b/test/c/CMakeLists.txt index dfa65da..f02cff8 100644 --- a/test/c/CMakeLists.txt +++ b/test/c/CMakeLists.txt @@ -5,6 +5,13 @@ add_executable(dvec_shape_test dvec_shape_test.c) target_link_libraries(dvec_shape_test binsparse) +add_executable(version_test version_test.c) +target_link_libraries(version_test binsparse) + add_test( NAME unit.dvec_shape COMMAND dvec_shape_test ${CMAKE_CURRENT_BINARY_DIR}/dvec_shape_test.h5) + +add_test( + NAME unit.version + COMMAND version_test ${CMAKE_CURRENT_BINARY_DIR}/version_test.h5) diff --git a/test/c/dvec_shape_test.c b/test/c/dvec_shape_test.c index 1e0914a..d8b1f2c 100644 --- a/test/c/dvec_shape_test.c +++ b/test/c/dvec_shape_test.c @@ -52,7 +52,7 @@ static void check_read(const char* filename, bool parallel) { static void create_invalid_shape(const char* filename) { static const char* json = - "{\"binsparse\":{\"version\":\"0.1\",\"format\":\"DVEC\"," + "{\"binsparse\":{\"version\":\"0.1.0\",\"format\":\"DVEC\"," "\"shape\":[3,1],\"number_of_stored_values\":3," "\"data_types\":{\"values\":\"float64\"}}}"; diff --git a/test/c/version_test.c b/test/c/version_test.c new file mode 100644 index 0000000..ef2e49f --- /dev/null +++ b/test/c/version_test.c @@ -0,0 +1,88 @@ +/* + * SPDX-FileCopyrightText: 2026 Binsparse Developers + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include +#include +#include +#include + +static void write_versioned_dvec(const char* filename, const char* version) { + unlink(filename); + + bsp_matrix_t matrix; + bsp_construct_default_matrix_t(&matrix); + matrix.format = BSP_DVEC; + matrix.structure = BSP_GENERAL; + matrix.nrows = 2; + matrix.ncols = 1; + matrix.nnz = 2; + assert(bsp_construct_array_t(&matrix.values, 2, BSP_FLOAT64) == BSP_SUCCESS); + double* values = matrix.values.data; + values[0] = 1; + values[1] = 2; + + assert(bsp_write_matrix(filename, matrix, NULL, NULL, 0) == BSP_SUCCESS); + bsp_destroy_matrix_t(&matrix); + + char json[1024]; + int length = snprintf( + json, sizeof(json), + "{\"binsparse\":{\"version\":\"%s\",\"format\":\"DVEC\"," + "\"shape\":[2],\"number_of_stored_values\":2," + "\"data_types\":{\"values\":\"float64\"}}}", + version); + assert(length > 0 && (size_t) length < sizeof(json)); + + hid_t file = H5Fopen(filename, H5F_ACC_RDWR, H5P_DEFAULT); + assert(file >= 0); + H5Adelete(file, "binsparse"); + assert(bsp_write_attribute(file, "binsparse", json) == BSP_SUCCESS); + H5Fclose(file); +} + +static void check_matrix_reader(const char* filename, const char* version, + bsp_error_t expected) { + write_versioned_dvec(filename, version); + + bsp_matrix_t matrix; + bsp_error_t error = bsp_read_matrix(&matrix, filename, NULL); + assert(error == expected); + if (error == BSP_SUCCESS) { + bsp_destroy_matrix_t(&matrix); + } + + unlink(filename); +} + +int main(int argc, char** argv) { + assert(argc == 2); + const char* filename = argv[1]; + + assert(bsp_check_version_compatible("0.1.0") == BSP_SUCCESS); + assert(bsp_check_version_compatible("0.1.0+roundtrip.1") == BSP_SUCCESS); + assert(bsp_check_version_compatible("0.1.1") == BSP_ERROR_UNSUPPORTED); + assert(bsp_check_version_compatible("0.1.42+roundtrip.1") == + BSP_ERROR_UNSUPPORTED); + assert(bsp_check_version_compatible("0.1.0-alpha.1") == + BSP_ERROR_UNSUPPORTED); + assert(bsp_check_version_compatible("0.2.0-alpha.1") == + BSP_ERROR_UNSUPPORTED); + assert(bsp_check_version_compatible("0.2.0") == BSP_ERROR_UNSUPPORTED); + assert(bsp_check_version_compatible("1.0.0") == BSP_ERROR_UNSUPPORTED); + assert(bsp_check_version_compatible("0.1") == BSP_ERROR_FORMAT); + assert(bsp_check_version_compatible("01.1.0") == BSP_ERROR_FORMAT); + assert(bsp_check_version_compatible(NULL) == BSP_ERROR_FORMAT); + + check_matrix_reader(filename, "0.1.0", BSP_SUCCESS); + check_matrix_reader(filename, "0.1.0+roundtrip.1", BSP_SUCCESS); + check_matrix_reader(filename, "0.1.7", BSP_ERROR_UNSUPPORTED); + check_matrix_reader(filename, "0.2.0-alpha.1", BSP_ERROR_UNSUPPORTED); + check_matrix_reader(filename, "0.2.0", BSP_ERROR_UNSUPPORTED); + check_matrix_reader(filename, "1.0.0", BSP_ERROR_UNSUPPORTED); + check_matrix_reader(filename, "0.1", BSP_ERROR_FORMAT); + + return 0; +} From d5c4cc798887d9d6c3330357bf90e388735434ab Mon Sep 17 00:00:00 2001 From: Willow Ahrens Date: Tue, 1 Sep 2026 21:12:59 -0400 Subject: [PATCH 11/16] fix --- CMakeLists.txt | 26 ++--- examples/CMakeLists.txt | 2 - include/binsparse/array.h | 4 - include/binsparse/binsparse_cJSON.h | 13 +++ include/binsparse/detail/allocator.h | 3 +- include/binsparse/detail/cpp/array.hpp | 133 ------------------------- include/binsparse/write_tensor.h | 13 ++- src/CMakeLists.txt | 4 +- src/version.c | 46 +++++++++ src/version.cpp | 47 --------- src/write_tensor.c | 66 ++++++++++-- test/CMakeLists.txt | 1 - test/bash/CMakeLists.txt | 6 -- test/c/version_test.c | 9 +- test/compliance/CMakeLists.txt | 1 + test/compliance/npy_to_binsparse.c | 4 +- test/compliance/reformat.c | 3 +- test/julia/CMakeLists.txt | 13 --- test/julia/tensor_test.jl | 95 ------------------ 19 files changed, 147 insertions(+), 342 deletions(-) delete mode 100644 include/binsparse/detail/cpp/array.hpp create mode 100644 src/version.c delete mode 100644 src/version.cpp delete mode 100644 test/julia/CMakeLists.txt delete mode 100644 test/julia/tensor_test.jl diff --git a/CMakeLists.txt b/CMakeLists.txt index e997ee4..2e060fc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,17 +3,15 @@ # SPDX-License-Identifier: BSD-3-Clause cmake_minimum_required(VERSION 3.20) -project(binsparse VERSION 1.0.0) +project(binsparse VERSION 1.0.0 LANGUAGES C) cmake_policy(SET CMP0079 NEW) set(CMAKE_C_STANDARD 11) -set(CMAKE_CXX_STANDARD 20) # FIXME: -march=native is not portable # set(CMAKE_C_FLAGS "-O3 -march=native") set(CMAKE_C_FLAGS "-g ") -set(CMAKE_CXX_FLAGS "-g ") option(ENABLE_SANITIZERS "Enable Clang sanitizers" OFF) @@ -48,15 +46,21 @@ FetchContent_Declare( ) FetchContent_MakeAvailable(cJSON) -# cpp-semver is header-only. Use strict SemVer parsing for file-format metadata. -set(SEMVER_BUILD_TESTS OFF CACHE INTERNAL "Disable cpp-semver tests") -set(SEMVER_INSTALL OFF CACHE INTERNAL "Disable cpp-semver install") FetchContent_Declare( - cpp-semver - GIT_REPOSITORY https://github.com/z4kn4fein/cpp-semver.git - GIT_TAG v0.4.0 + semver_c + GIT_REPOSITORY https://github.com/h2non/semver.c.git + GIT_TAG v1.0.0 ) -FetchContent_MakeAvailable(cpp-semver) +FetchContent_MakeAvailable(semver_c) +add_library(binsparse_semver OBJECT ${semver_c_SOURCE_DIR}/semver.c) +set_target_properties(binsparse_semver PROPERTIES + POSITION_INDEPENDENT_CODE ON) +target_include_directories(binsparse_semver PRIVATE + $) +target_sources(binsparse PRIVATE $) +target_sources(binsparse_dynamic PRIVATE $) +target_include_directories(binsparse PRIVATE ${semver_c_SOURCE_DIR}) +target_include_directories(binsparse_dynamic PRIVATE ${semver_c_SOURCE_DIR}) # Restore original BUILD_SHARED_LIBS setting set(BUILD_SHARED_LIBS ${BUILD_SHARED_LIBS_BACKUP}) @@ -64,8 +68,6 @@ set(BUILD_SHARED_LIBS ${BUILD_SHARED_LIBS_BACKUP}) configure_file(${cJSON_SOURCE_DIR}/cJSON.h ${CMAKE_BINARY_DIR}/include/cJSON/cJSON.h) target_link_libraries(binsparse PRIVATE cjson) target_link_libraries(binsparse_dynamic PRIVATE cjson) -target_link_libraries(binsparse PRIVATE $) -target_link_libraries(binsparse_dynamic PRIVATE $) # Set up include directories properly for both build and install target_include_directories(${PROJECT_NAME} diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index e090d7c..0c9ce2c 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -20,5 +20,3 @@ add_example(benchmark_read) add_example(benchmark_read_parallel) add_example(benchmark_write) add_example(tensor_test) - -add_subdirectory(cpp) diff --git a/include/binsparse/array.h b/include/binsparse/array.h index 02b063c..be0e3d3 100644 --- a/include/binsparse/array.h +++ b/include/binsparse/array.h @@ -402,7 +402,3 @@ static inline bool bsp_array_equal(bsp_array_t x, bsp_array_t y) { } #endif - -#ifdef __cplusplus -#include -#endif diff --git a/include/binsparse/binsparse_cJSON.h b/include/binsparse/binsparse_cJSON.h index e72b065..19c2513 100644 --- a/include/binsparse/binsparse_cJSON.h +++ b/include/binsparse/binsparse_cJSON.h @@ -8,6 +8,7 @@ #include #include +#include #include @@ -25,12 +26,24 @@ bsp_error_t bsp_write_matrix_cjson(const char* fname, bsp_matrix_t matrix, const char* group, cJSON* user_json, int compression_level); +bsp_error_t bsp_write_tensor_cjson(const char* fname, bsp_tensor_t tensor, + const char* group, cJSON* user_json, + int compression_level); + #ifdef BSP_BINSPARSE_HDF5_H bsp_error_t bsp_write_matrix_to_group_cjson(hid_t f, bsp_matrix_t matrix, cJSON* user_json, int compression_level); #endif +#ifdef BSP_USE_HDF5 +#include + +bsp_error_t bsp_write_tensor_to_group_cjson(hid_t f, bsp_tensor_t tensor, + cJSON* user_json, + int compression_level); +#endif + #ifdef __cplusplus } #endif diff --git a/include/binsparse/detail/allocator.h b/include/binsparse/detail/allocator.h index 86d817a..0576a61 100644 --- a/include/binsparse/detail/allocator.h +++ b/include/binsparse/detail/allocator.h @@ -13,5 +13,4 @@ typedef struct bsp_allocator_t { void (*free)(void*); } bsp_allocator_t; -const static bsp_allocator_t bsp_default_allocator = {.malloc = malloc, - .free = free}; +const static bsp_allocator_t bsp_default_allocator = {malloc, free}; diff --git a/include/binsparse/detail/cpp/array.hpp b/include/binsparse/detail/cpp/array.hpp deleted file mode 100644 index 9842cef..0000000 --- a/include/binsparse/detail/cpp/array.hpp +++ /dev/null @@ -1,133 +0,0 @@ -// SPDX-FileCopyrightText: 2024 Binsparse Developers -// -// SPDX-License-Identifier: BSD-3-Clause - -#pragma once - -#ifdef __cplusplus - -#include -#include -#include - -namespace binsparse { - -namespace __detail { - -using array_ptr_variant_t = - std::variant; - -inline array_ptr_variant_t get_typed_ptr(bsp_array_t array) { - if (array.type == BSP_UINT8) { - uint8_t* data = (uint8_t*) array.data; - return data; - } else if (array.type == BSP_UINT16) { - uint16_t* data = (uint16_t*) array.data; - return data; - } else if (array.type == BSP_UINT32) { - uint32_t* data = (uint32_t*) array.data; - return data; - } else if (array.type == BSP_UINT64) { - uint64_t* data = (uint64_t*) array.data; - return data; - } else if (array.type == BSP_INT8) { - int8_t* data = (int8_t*) array.data; - return data; - } else if (array.type == BSP_INT16) { - int16_t* data = (int16_t*) array.data; - return data; - } else if (array.type == BSP_INT32) { - int32_t* data = (int32_t*) array.data; - return data; - } else if (array.type == BSP_INT64) { - int64_t* data = (int64_t*) array.data; - return data; - } else if (array.type == BSP_FLOAT32) { - float* data = (float*) array.data; - return data; - } else if (array.type == BSP_FLOAT64) { - double* data = (double*) array.data; - return data; - } else if (array.type == BSP_BINT8) { - int8_t* data = (int8_t*) array.data; - return data; - } else if (array.type == BSP_COMPLEX_FLOAT32) { - float _Complex* data = (float _Complex*) array.data; - return data; - } else if (array.type == BSP_COMPLEX_FLOAT64) { - double _Complex* data = (double _Complex*) array.data; - return data; - } - return {}; -} - -template -constexpr static bool is_c_complex_v = - std::is_same_v, _Complex double> || - std::is_same_v, _Complex float>; - -template -constexpr static bool mixed_c_complex_v = - (is_c_complex_v && !is_c_complex_v) || - (!is_c_complex_v && is_c_complex_v); - -} // namespace __detail - -} // namespace binsparse - -// value = array[index] -template -inline void bsp_array_read(bsp_array_t array, size_t index, T& value) { - auto variant_ptr = binsparse::__detail::get_typed_ptr(array); - - std::visit( - [&](auto* ptr) { - using U = std::remove_pointer_t; - - if constexpr (std::is_convertible_v && - !binsparse::__detail::mixed_c_complex_v) { - value = ptr[index]; - } - }, - variant_ptr); -} - -// array[index] = value -template -inline void bsp_array_write(bsp_array_t array, size_t index, U value) { - auto variant_ptr = binsparse::__detail::get_typed_ptr(array); - - std::visit( - [&](auto* ptr) { - using T = std::remove_pointer_t; - - if constexpr (std::is_convertible_v && - !binsparse::__detail::mixed_c_complex_v) { - ptr[index] = value; - } - }, - variant_ptr); -} - -// array_0[index_0] = array_1[index_1] -inline void bsp_array_awrite(bsp_array_t array_0, size_t index_0, - bsp_array_t array_1, size_t index_1) { - auto variant_ptr_0 = binsparse::__detail::get_typed_ptr(array_0); - auto variant_ptr_1 = binsparse::__detail::get_typed_ptr(array_1); - - std::visit( - [&](auto* ptr_0, auto* ptr_1) { - using T = std::remove_pointer_t; - using U = std::remove_pointer_t; - - if constexpr (std::is_convertible_v && - !binsparse::__detail::mixed_c_complex_v) { - ptr_0[index_0] = ptr_1[index_1]; - } - }, - variant_ptr_0, variant_ptr_1); -} - -#endif diff --git a/include/binsparse/write_tensor.h b/include/binsparse/write_tensor.h index 27e1f29..feef5f3 100644 --- a/include/binsparse/write_tensor.h +++ b/include/binsparse/write_tensor.h @@ -10,20 +10,19 @@ extern "C" { #endif -// TODO: make cJSON optional. - #include -#include #ifdef BSP_USE_HDF5 #include -int bsp_write_tensor_to_group(hid_t f, bsp_tensor_t tensor, cJSON* user_json, - int compression_level); +bsp_error_t bsp_write_tensor_to_group(hid_t f, bsp_tensor_t tensor, + const char* user_json, + int compression_level); #endif -int bsp_write_tensor(const char* fname, bsp_tensor_t tensor, const char* group, - cJSON* user_json, int compression_level); +bsp_error_t bsp_write_tensor(const char* fname, bsp_tensor_t tensor, + const char* group, const char* user_json, + int compression_level); #ifdef __cplusplus } diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 1afd501..f63da43 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -5,7 +5,7 @@ target_sources(binsparse PRIVATE read_matrix.c read_tensor.c - version.cpp + version.c write_matrix.c write_tensor.c ) @@ -13,7 +13,7 @@ target_sources(binsparse PRIVATE target_sources(binsparse_dynamic PRIVATE read_matrix.c read_tensor.c - version.cpp + version.c write_matrix.c write_tensor.c ) diff --git a/src/version.c b/src/version.c new file mode 100644 index 0000000..bea2bfa --- /dev/null +++ b/src/version.c @@ -0,0 +1,46 @@ +/* + * SPDX-FileCopyrightText: 2026 Binsparse Developers + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include + +#include + +#include + +static bool parse_version(const char* text, semver_t* version) { + return text != NULL && semver_parse(text, version) == 0; +} + +static bool compatible(const semver_t* version, const semver_t* supported) { + if (version->major != supported->major) + return false; + + if (supported->major == 0) + return semver_compare_version(*version, *supported) == 0; + + return version->minor >= supported->minor; +} + +bsp_error_t bsp_check_version_compatible(const char* version) { + semver_t parsed = {0}; + semver_t supported = {0}; + bsp_error_t error = BSP_SUCCESS; + bool parsed_ok = parse_version(version, &parsed); + bool supported_ok = parsed_ok && parse_version(BINSPARSE_VERSION, &supported); + + if (!parsed_ok || !supported_ok) { + semver_free(&parsed); + semver_free(&supported); + return BSP_ERROR_FORMAT; + } + + if (!compatible(&parsed, &supported)) + error = BSP_ERROR_UNSUPPORTED; + + semver_free(&parsed); + semver_free(&supported); + return error; +} diff --git a/src/version.cpp b/src/version.cpp deleted file mode 100644 index 71595e0..0000000 --- a/src/version.cpp +++ /dev/null @@ -1,47 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2026 Binsparse Developers - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -#include -#include - -#include - -namespace { - -bool bsp_semver_compatible(const semver::version& version, - const semver::version& supported) { - // Mirror python-semver Version.is_compatible(). - if (supported.major() == 0 && version.major() == 0 && - (supported.minor() != version.minor() || - supported.patch() != version.patch() || - supported.prerelease() != version.prerelease())) { - return false; - } - - return supported.major() == version.major() && - version.minor() >= supported.minor() && - supported.prerelease() == version.prerelease(); -} - -} // namespace - -extern "C" bsp_error_t bsp_check_version_compatible(const char* version) { - if (version == nullptr) { - return BSP_ERROR_FORMAT; - } - - try { - semver::version parsed = semver::version::parse(version); - semver::version supported = semver::version::parse(BINSPARSE_VERSION); - if (!bsp_semver_compatible(parsed, supported)) { - return BSP_ERROR_UNSUPPORTED; - } - } catch (const std::exception&) { - return BSP_ERROR_FORMAT; - } - - return BSP_SUCCESS; -} diff --git a/src/write_tensor.c b/src/write_tensor.c index 1b71f17..27751fe 100644 --- a/src/write_tensor.c +++ b/src/write_tensor.c @@ -5,8 +5,10 @@ */ #include +#include #include #include +#include #include #include @@ -27,10 +29,12 @@ static cJSON* init_tensor_json(bsp_tensor_t tensor, cJSON* user_json) { cJSON_AddItemToObject(binsparse, "custom", binsparse_custom); cJSON_AddItemToObject(j, "binsparse", binsparse); - cJSON* userJsonItem; - - cJSON_ArrayForEach(userJsonItem, user_json) { - cJSON_AddItemToObject(j, userJsonItem->string, userJsonItem); + if (user_json != NULL) { + cJSON* user_json_item; + cJSON_ArrayForEach(user_json_item, user_json) { + cJSON* item_copy = cJSON_Duplicate(user_json_item, 1); + cJSON_AddItemToObject(j, user_json_item->string, item_copy); + } } cJSON_AddStringToObject(binsparse, "version", BINSPARSE_VERSION); @@ -55,8 +59,9 @@ static cJSON* init_tensor_json(bsp_tensor_t tensor, cJSON* user_json) { return j; } -bsp_error_t bsp_write_tensor_to_group(hid_t f, bsp_tensor_t tensor, - cJSON* user_json, int compression_level) { +bsp_error_t bsp_write_tensor_to_group_cjson(hid_t f, bsp_tensor_t tensor, + cJSON* user_json, + int compression_level) { // bsp_matrix_t matrix; cJSON* j = init_tensor_json(tensor, user_json); // tensor: @@ -167,13 +172,29 @@ bsp_error_t bsp_write_tensor_to_group(hid_t f, bsp_tensor_t tensor, return BSP_SUCCESS; } -bsp_error_t bsp_write_tensor(const char* fname, bsp_tensor_t tensor, - const char* group, cJSON* user_json, - int compression_level) { +bsp_error_t bsp_write_tensor_to_group(hid_t f, bsp_tensor_t tensor, + const char* user_json, + int compression_level) { + cJSON* user_json_cjson = NULL; + if (user_json != NULL) { + user_json_cjson = cJSON_Parse(user_json); + } + if (user_json_cjson == NULL) { + user_json_cjson = cJSON_CreateObject(); + } + bsp_error_t error = bsp_write_tensor_to_group_cjson( + f, tensor, user_json_cjson, compression_level); + cJSON_Delete(user_json_cjson); + return error; +} + +bsp_error_t bsp_write_tensor_cjson(const char* fname, bsp_tensor_t tensor, + const char* group, cJSON* user_json, + int compression_level) { if (group == NULL) { hid_t f = H5Fcreate(fname, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); bsp_error_t error = - bsp_write_tensor_to_group(f, tensor, user_json, compression_level); + bsp_write_tensor_to_group_cjson(f, tensor, user_json, compression_level); if (error != BSP_SUCCESS) { H5Fclose(f); return error; @@ -187,9 +208,32 @@ bsp_error_t bsp_write_tensor(const char* fname, bsp_tensor_t tensor, f = H5Fcreate(fname, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); } hid_t g = H5Gcreate1(f, group, H5P_DEFAULT); - bsp_write_tensor_to_group(g, tensor, user_json, compression_level); + bsp_error_t error = + bsp_write_tensor_to_group_cjson(g, tensor, user_json, compression_level); + if (error != BSP_SUCCESS) { + H5Gclose(g); + H5Fclose(f); + return error; + } H5Gclose(g); H5Fclose(f); } return BSP_SUCCESS; } + +bsp_error_t bsp_write_tensor(const char* fname, bsp_tensor_t tensor, + const char* group, const char* user_json, + int compression_level) { + cJSON* user_json_cjson = NULL; + if (user_json != NULL) { + user_json_cjson = cJSON_Parse(user_json); + } + if (user_json_cjson == NULL) { + user_json_cjson = cJSON_CreateObject(); + } + + bsp_error_t error = bsp_write_tensor_cjson( + fname, tensor, group, user_json_cjson, compression_level); + cJSON_Delete(user_json_cjson); + return error; +} diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index bc81d56..abc3905 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -4,4 +4,3 @@ add_subdirectory(bash) add_subdirectory(c) -add_subdirectory(julia) diff --git a/test/bash/CMakeLists.txt b/test/bash/CMakeLists.txt index dc6ba32..7586317 100644 --- a/test/bash/CMakeLists.txt +++ b/test/bash/CMakeLists.txt @@ -35,10 +35,4 @@ if(BASH_PROGRAM) add_test(NAME integration.mouse_gene COMMAND ${BASH_PROGRAM} ${CMAKE_CURRENT_SOURCE_DIR}/test.sh ${CMAKE_BINARY_DIR}/data/mouse_gene.tar.gz) add_test(NAME integration.IMDB COMMAND ${BASH_PROGRAM} ${CMAKE_CURRENT_SOURCE_DIR}/test.sh ${CMAKE_BINARY_DIR}/data/IMDB.tar.gz) add_test(NAME integration.dwg961a COMMAND ${BASH_PROGRAM} ${CMAKE_CURRENT_SOURCE_DIR}/test.sh ${CMAKE_BINARY_DIR}/data/dwg961a.tar.gz) - - add_test(NAME integration.cpp.chesapeake COMMAND ${BASH_PROGRAM} ${CMAKE_CURRENT_SOURCE_DIR}/test-cpp.sh ${CMAKE_BINARY_DIR}/data/chesapeake.tar.gz) - add_test(NAME integration.cpp.1138_bus COMMAND ${BASH_PROGRAM} ${CMAKE_CURRENT_SOURCE_DIR}/test-cpp.sh ${CMAKE_BINARY_DIR}/data/1138_bus.tar.gz) - add_test(NAME integration.cpp.mouse_gene COMMAND ${BASH_PROGRAM} ${CMAKE_CURRENT_SOURCE_DIR}/test-cpp.sh ${CMAKE_BINARY_DIR}/data/mouse_gene.tar.gz) - add_test(NAME integration.cpp.IMDB COMMAND ${BASH_PROGRAM} ${CMAKE_CURRENT_SOURCE_DIR}/test-cpp.sh ${CMAKE_BINARY_DIR}/data/IMDB.tar.gz) - add_test(NAME integration.cpp.dwg961a COMMAND ${BASH_PROGRAM} ${CMAKE_CURRENT_SOURCE_DIR}/test-cpp.sh ${CMAKE_BINARY_DIR}/data/dwg961a.tar.gz) endif() diff --git a/test/c/version_test.c b/test/c/version_test.c index ef2e49f..8093638 100644 --- a/test/c/version_test.c +++ b/test/c/version_test.c @@ -66,23 +66,22 @@ int main(int argc, char** argv) { assert(bsp_check_version_compatible("0.1.1") == BSP_ERROR_UNSUPPORTED); assert(bsp_check_version_compatible("0.1.42+roundtrip.1") == BSP_ERROR_UNSUPPORTED); - assert(bsp_check_version_compatible("0.1.0-alpha.1") == - BSP_ERROR_UNSUPPORTED); + assert(bsp_check_version_compatible("0.1.0-alpha.1") == BSP_SUCCESS); assert(bsp_check_version_compatible("0.2.0-alpha.1") == BSP_ERROR_UNSUPPORTED); assert(bsp_check_version_compatible("0.2.0") == BSP_ERROR_UNSUPPORTED); assert(bsp_check_version_compatible("1.0.0") == BSP_ERROR_UNSUPPORTED); - assert(bsp_check_version_compatible("0.1") == BSP_ERROR_FORMAT); - assert(bsp_check_version_compatible("01.1.0") == BSP_ERROR_FORMAT); + assert(bsp_check_version_compatible("not-a-version") == BSP_ERROR_FORMAT); assert(bsp_check_version_compatible(NULL) == BSP_ERROR_FORMAT); check_matrix_reader(filename, "0.1.0", BSP_SUCCESS); check_matrix_reader(filename, "0.1.0+roundtrip.1", BSP_SUCCESS); check_matrix_reader(filename, "0.1.7", BSP_ERROR_UNSUPPORTED); + check_matrix_reader(filename, "0.1.0-alpha.1", BSP_SUCCESS); check_matrix_reader(filename, "0.2.0-alpha.1", BSP_ERROR_UNSUPPORTED); check_matrix_reader(filename, "0.2.0", BSP_ERROR_UNSUPPORTED); check_matrix_reader(filename, "1.0.0", BSP_ERROR_UNSUPPORTED); - check_matrix_reader(filename, "0.1", BSP_ERROR_FORMAT); + check_matrix_reader(filename, "not-a-version", BSP_ERROR_FORMAT); return 0; } diff --git a/test/compliance/CMakeLists.txt b/test/compliance/CMakeLists.txt index a3a639c..0ea277f 100644 --- a/test/compliance/CMakeLists.txt +++ b/test/compliance/CMakeLists.txt @@ -5,6 +5,7 @@ SPDX-License-Identifier: BSD-3-Clause ]] find_package(ZLIB REQUIRED) +enable_language(CXX) add_library(cnpy_vendor STATIC ${PROJECT_SOURCE_DIR}/third_party/cnpy/cnpy.cpp diff --git a/test/compliance/npy_to_binsparse.c b/test/compliance/npy_to_binsparse.c index 9ec16ca..f5f9b50 100644 --- a/test/compliance/npy_to_binsparse.c +++ b/test/compliance/npy_to_binsparse.c @@ -7,6 +7,8 @@ #include "cnpy_c.h" #include "reformat.h" +#include + #include #include #include @@ -158,7 +160,7 @@ int main(int argc, char** argv) { root_ptr, 1, true, values); if (!strcmp(format, "custom")) { cJSON* user = cJSON_CreateObject(); - if (bsp_write_tensor(argv[5], result, NULL, user, 0) != BSP_SUCCESS) + if (bsp_write_tensor_cjson(argv[5], result, NULL, user, 0) != BSP_SUCCESS) die("cannot write Binsparse tensor"); cJSON_Delete(user); } else if (write_predefined(argv[5], result, format, 0) != BSP_SUCCESS) { diff --git a/test/compliance/reformat.c b/test/compliance/reformat.c index f3cf184..ca0bd30 100644 --- a/test/compliance/reformat.c +++ b/test/compliance/reformat.c @@ -14,6 +14,7 @@ #include "reformat.h" +#include #include #include #include @@ -783,7 +784,7 @@ bsp_error_t bsp_reformat_file(const char* input, const char* output, root_ptr, 1, true, values); if (!strcmp(target_format, "custom")) { cJSON* user = cJSON_CreateObject(); - if (bsp_write_tensor(output, result, NULL, user, compression) != + if (bsp_write_tensor_cjson(output, result, NULL, user, compression) != BSP_SUCCESS) die("failed to write output tensor"); cJSON_Delete(user); diff --git a/test/julia/CMakeLists.txt b/test/julia/CMakeLists.txt deleted file mode 100644 index 4c2c9d8..0000000 --- a/test/julia/CMakeLists.txt +++ /dev/null @@ -1,13 +0,0 @@ -# SPDX-FileCopyrightText: 2024 Binsparse Developers -# -# SPDX-License-Identifier: BSD-3-Clause - -find_program(JULIA_PROGRAM julia) - -enable_testing() - -if(JULIA_PROGRAM) - add_test(NAME tensors.tensor_test COMMAND ${JULIA_PROGRAM} ${CMAKE_CURRENT_SOURCE_DIR}/tensor_test.jl) - - add_test(NAME tensors.cpp.tensor_test COMMAND ${JULIA_PROGRAM} ${CMAKE_CURRENT_SOURCE_DIR}/tensor_test.jl cpp) -endif() diff --git a/test/julia/tensor_test.jl b/test/julia/tensor_test.jl deleted file mode 100644 index cbba45f..0000000 --- a/test/julia/tensor_test.jl +++ /dev/null @@ -1,95 +0,0 @@ -# SPDX-FileCopyrightText: 2024 Binsparse Developers -# -# SPDX-License-Identifier: BSD-3-Clause - -using Finch; -using HDF5; - -dir = pwd(); - -if length(ARGS) == 0 - tensor_test = joinpath(dir, "../../examples/tensor_test") - test_files = joinpath(dir, "../../tensor_test_files") -else - tensor_test = joinpath(dir, "../../examples/cpp/tensor_test-cpp") - test_files = joinpath(dir, "../../tensor_test_files/cpp") -end - -mkpath(test_files) - -function tensortest(tensor::Tensor, input::AbstractString, output::AbstractString) - print("tensor_test ", input, " -> ", output, "\n") - fwrite(input, tensor) - run(`$tensor_test $input $output`) - output_tensor = fread(output) - @assert tensor == output_tensor -end - -tensortest( - Tensor( - Dense(SparseList{Int32}(SparseList{Int32}(Element{0.0,Float64,Int32}()))), - fsprand(10, 10, 10, 0.1) - ), - joinpath(test_files, "input1.bsp.h5"), - joinpath(test_files, "output1.bsp.h5") -) - - -tensortest( - Tensor( - Dense(SparseCOO{2}(Element(0.0))), - fsprand(10, 10, 10, 0.1) - ), - joinpath(test_files, "input2.bsp.h5"), - joinpath(test_files, "output2.bsp.h5") -) - -tensortest( - Tensor( - Dense(Dense(Dense(Element{0.0,Float64,Int32}()))), - fsprand(10, 10, 10, 0.1) - ), - joinpath(test_files, "input3.bsp.h5"), - joinpath(test_files, "output3.bsp.h5") -) - -tensortest( - Tensor( - SparseCOO{2}(Element(0.0)), - fsprand(10, 10, 0.1) - ), - joinpath(test_files, "input4.bsp.h5"), - joinpath(test_files, "output4.bsp.h5") -) - -tensortest( - Tensor( - Dense(SparseList{Int32}(Element{0.0,Float64,Int32}())), - [ - 0 1 0 3; - 1 0 0 4; - ] - ), - joinpath(test_files, "inputcsr.bsp.h5"), - joinpath(test_files, "outputcsr.bsp.h5") -) -tensortest( - Tensor( - Dense(SparseList{Int32}(Element{0.0,Float64,Int32}())), - fsprand(10, 10, 0.1) - ), - joinpath(test_files, "inputcsr2.bsp.h5"), - joinpath(test_files, "outputcsr2.bsp.h5") -) - -tensortest( - Tensor( - Dense(Dense(Element(0.0))), - [ - 1 1 2 3; - 6 1 5 4; - ] - ), - joinpath(test_files, "inputdense.bsp.h5"), - joinpath(test_files, "outputdense.bsp.h5") -) From 5ac2980334bffdfcb142c6e3ca12b923ae1b02b4 Mon Sep 17 00:00:00 2001 From: Willow Ahrens Date: Tue, 1 Sep 2026 21:16:21 -0400 Subject: [PATCH 12/16] solid --- test/compliance/CMakeLists.txt | 13 +- third_party/cnpy/CMakeLists.txt | 36 --- third_party/cnpy/LICENSE | 21 -- third_party/cnpy/README.md | 61 ----- third_party/cnpy/cnpy.cpp | 386 -------------------------------- third_party/cnpy/cnpy.h | 310 ------------------------- third_party/cnpy/example1.cpp | 64 ------ third_party/cnpy/mat2npz | 22 -- third_party/cnpy/npy2mat | 19 -- third_party/cnpy/npz2mat | 19 -- 10 files changed, 11 insertions(+), 940 deletions(-) delete mode 100644 third_party/cnpy/CMakeLists.txt delete mode 100644 third_party/cnpy/LICENSE delete mode 100644 third_party/cnpy/README.md delete mode 100644 third_party/cnpy/cnpy.cpp delete mode 100644 third_party/cnpy/cnpy.h delete mode 100644 third_party/cnpy/example1.cpp delete mode 100644 third_party/cnpy/mat2npz delete mode 100644 third_party/cnpy/npy2mat delete mode 100755 third_party/cnpy/npz2mat diff --git a/test/compliance/CMakeLists.txt b/test/compliance/CMakeLists.txt index 0ea277f..e59d498 100644 --- a/test/compliance/CMakeLists.txt +++ b/test/compliance/CMakeLists.txt @@ -6,13 +6,22 @@ SPDX-License-Identifier: BSD-3-Clause find_package(ZLIB REQUIRED) enable_language(CXX) +include(FetchContent) + +FetchContent_Declare( + cnpy + GIT_REPOSITORY https://github.com/rogersce/cnpy.git + GIT_TAG 4e8810b1a8637695171ed346ce68f6984e585ef4 + SOURCE_SUBDIR cmake +) +FetchContent_MakeAvailable(cnpy) add_library(cnpy_vendor STATIC - ${PROJECT_SOURCE_DIR}/third_party/cnpy/cnpy.cpp + ${cnpy_SOURCE_DIR}/cnpy.cpp cnpy_c.cpp ) target_include_directories(cnpy_vendor PUBLIC - ${PROJECT_SOURCE_DIR}/third_party/cnpy + ${cnpy_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR} ) target_link_libraries(cnpy_vendor PUBLIC ZLIB::ZLIB) diff --git a/third_party/cnpy/CMakeLists.txt b/third_party/cnpy/CMakeLists.txt deleted file mode 100644 index 6fd1d65..0000000 --- a/third_party/cnpy/CMakeLists.txt +++ /dev/null @@ -1,36 +0,0 @@ -#[[ -SPDX-FileCopyrightText: 2011 Carl Rogers - -SPDX-License-Identifier: MIT -]] - -CMAKE_MINIMUM_REQUIRED(VERSION 3.0 FATAL_ERROR) -if(COMMAND cmake_policy) - cmake_policy(SET CMP0003 NEW) -endif(COMMAND cmake_policy) - -project(CNPY) - -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11") - -option(ENABLE_STATIC "Build static (.a) library" ON) - -find_package(ZLIB REQUIRED) - -include_directories(${ZLIB_INCLUDE_DIRS}) - -add_library(cnpy SHARED "cnpy.cpp") -target_link_libraries(cnpy ${ZLIB_LIBRARIES}) -install(TARGETS "cnpy" LIBRARY DESTINATION lib PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE) - -if(ENABLE_STATIC) - add_library(cnpy-static STATIC "cnpy.cpp") - set_target_properties(cnpy-static PROPERTIES OUTPUT_NAME "cnpy") - install(TARGETS "cnpy-static" ARCHIVE DESTINATION lib) -endif(ENABLE_STATIC) - -install(FILES "cnpy.h" DESTINATION include) -install(FILES "mat2npz" "npy2mat" "npz2mat" DESTINATION bin PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE) - -add_executable(example1 example1.cpp) -target_link_libraries(example1 cnpy) diff --git a/third_party/cnpy/LICENSE b/third_party/cnpy/LICENSE deleted file mode 100644 index e60eadb..0000000 --- a/third_party/cnpy/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -The MIT License - -Copyright (c) Carl Rogers, 2011 - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. diff --git a/third_party/cnpy/README.md b/third_party/cnpy/README.md deleted file mode 100644 index edef9e5..0000000 --- a/third_party/cnpy/README.md +++ /dev/null @@ -1,61 +0,0 @@ - - -# Purpose: - -NumPy offers the `save` method for easy saving of arrays into .npy and `savez` for zipping multiple .npy arrays together into a .npz file. - -`cnpy` lets you read and write to these formats in C++. - -The motivation comes from scientific programming where large amounts of data are generated in C++ and analyzed in Python. - -Writing to .npy has the advantage of using low-level C++ I/O (fread and fwrite) for speed and binary format for size. -The .npy file header takes care of specifying the size, shape, and data type of the array, so specifying the format of the data is unnecessary. - -Loading data written in numpy formats into C++ is equally simple, but requires you to type-cast the loaded data to the type of your choice. - -# Installation: - -Default installation directory is /usr/local. -To specify a different directory, add `-DCMAKE_INSTALL_PREFIX=/path/to/install/dir` to the cmake invocation in step 4. - -1. get [cmake](www.cmake.org) -2. create a build directory, say $HOME/build -3. cd $HOME/build -4. cmake /path/to/cnpy -5. make -6. make install - -# Using: - -To use, `#include"cnpy.h"` in your source code. Compile the source code mycode.cpp as - -```bash -g++ -o mycode mycode.cpp -L/path/to/install/dir -lcnpy -lz --std=c++11 -``` - -# Description: - -There are two functions for writing data: `npy_save` and `npz_save`. - -There are 3 functions for reading: -- `npy_load` will load a .npy file. -- `npz_load(fname)` will load a .npz and return a dictionary of NpyArray structues. -- `npz_load(fname,varname)` will load and return the NpyArray for data varname from the specified .npz file. - -The data structure for loaded data is below. -Data is accessed via the `data()`-method, which returns a pointer of the specified type (which must match the underlying datatype of the data). -The array shape and word size are read from the npy header. - -```c++ -struct NpyArray { - std::vector shape; - size_t word_size; - template T* data(); -}; -``` - -See [example1.cpp](example1.cpp) for examples of how to use the library. example1 will also be build during cmake installation. diff --git a/third_party/cnpy/cnpy.cpp b/third_party/cnpy/cnpy.cpp deleted file mode 100644 index b275d5d..0000000 --- a/third_party/cnpy/cnpy.cpp +++ /dev/null @@ -1,386 +0,0 @@ -// SPDX-FileCopyrightText: 2011 Carl Rogers -// -// SPDX-License-Identifier: MIT - -// Copyright (C) 2011 Carl Rogers -// Released under MIT License -// license available in LICENSE file, or at -// http://www.opensource.org/licenses/mit-license.php - -#include "cnpy.h" -#include -#include -#include -#include -#include -#include -#include -#include - -char cnpy::BigEndianTest() { - int x = 1; - return (((char*) &x)[0]) ? '<' : '>'; -} - -char cnpy::map_type(const std::type_info& t) { - if (t == typeid(float)) - return 'f'; - if (t == typeid(double)) - return 'f'; - if (t == typeid(long double)) - return 'f'; - - if (t == typeid(int)) - return 'i'; - if (t == typeid(char)) - return 'i'; - if (t == typeid(signed char)) - return 'i'; - if (t == typeid(short)) - return 'i'; - if (t == typeid(long)) - return 'i'; - if (t == typeid(long long)) - return 'i'; - - if (t == typeid(unsigned char)) - return 'u'; - if (t == typeid(unsigned short)) - return 'u'; - if (t == typeid(unsigned long)) - return 'u'; - if (t == typeid(unsigned long long)) - return 'u'; - if (t == typeid(unsigned int)) - return 'u'; - - if (t == typeid(bool)) - return 'b'; - - if (t == typeid(std::complex)) - return 'c'; - if (t == typeid(std::complex)) - return 'c'; - if (t == typeid(std::complex)) - return 'c'; - - else - return '?'; -} - -template <> -std::vector& cnpy::operator+=(std::vector& lhs, - const std::string rhs) { - lhs.insert(lhs.end(), rhs.begin(), rhs.end()); - return lhs; -} - -template <> -std::vector& cnpy::operator+=(std::vector& lhs, const char* rhs) { - // write in little endian - size_t len = strlen(rhs); - lhs.reserve(len); - for (size_t byte = 0; byte < len; byte++) { - lhs.push_back(rhs[byte]); - } - return lhs; -} - -void cnpy::parse_npy_header(unsigned char* buffer, size_t& word_size, - std::vector& shape, bool& fortran_order) { - // std::string magic_string(buffer,6); - uint8_t major_version = *reinterpret_cast(buffer + 6); - uint8_t minor_version = *reinterpret_cast(buffer + 7); - uint16_t header_len = *reinterpret_cast(buffer + 8); - std::string header(reinterpret_cast(buffer + 9), header_len); - - size_t loc1, loc2; - - // fortran order - loc1 = header.find("fortran_order") + 16; - fortran_order = (header.substr(loc1, 4) == "True" ? true : false); - - // shape - loc1 = header.find("("); - loc2 = header.find(")"); - - std::regex num_regex("[0-9][0-9]*"); - std::smatch sm; - shape.clear(); - - std::string str_shape = header.substr(loc1 + 1, loc2 - loc1 - 1); - while (std::regex_search(str_shape, sm, num_regex)) { - shape.push_back(std::stoi(sm[0].str())); - str_shape = sm.suffix().str(); - } - - // endian, word size, data type - // byte order code | stands for not applicable. - // not sure when this applies except for byte array - loc1 = header.find("descr") + 9; - bool littleEndian = - (header[loc1] == '<' || header[loc1] == '|' ? true : false); - assert(littleEndian); - - // char type = header[loc1+1]; - // assert(type == map_type(T)); - - std::string str_ws = header.substr(loc1 + 2); - loc2 = str_ws.find("'"); - word_size = atoi(str_ws.substr(0, loc2).c_str()); -} - -void cnpy::parse_npy_header(FILE* fp, size_t& word_size, - std::vector& shape, bool& fortran_order) { - char buffer[256]; - size_t res = fread(buffer, sizeof(char), 11, fp); - if (res != 11) - throw std::runtime_error("parse_npy_header: failed fread"); - std::string header = fgets(buffer, 256, fp); - assert(header[header.size() - 1] == '\n'); - - size_t loc1, loc2; - - // fortran order - loc1 = header.find("fortran_order"); - if (loc1 == std::string::npos) - throw std::runtime_error( - "parse_npy_header: failed to find header keyword: 'fortran_order'"); - loc1 += 16; - fortran_order = (header.substr(loc1, 4) == "True" ? true : false); - - // shape - loc1 = header.find("("); - loc2 = header.find(")"); - if (loc1 == std::string::npos || loc2 == std::string::npos) - throw std::runtime_error( - "parse_npy_header: failed to find header keyword: '(' or ')'"); - - std::regex num_regex("[0-9][0-9]*"); - std::smatch sm; - shape.clear(); - - std::string str_shape = header.substr(loc1 + 1, loc2 - loc1 - 1); - while (std::regex_search(str_shape, sm, num_regex)) { - shape.push_back(std::stoi(sm[0].str())); - str_shape = sm.suffix().str(); - } - - // endian, word size, data type - // byte order code | stands for not applicable. - // not sure when this applies except for byte array - loc1 = header.find("descr"); - if (loc1 == std::string::npos) - throw std::runtime_error( - "parse_npy_header: failed to find header keyword: 'descr'"); - loc1 += 9; - bool littleEndian = - (header[loc1] == '<' || header[loc1] == '|' ? true : false); - assert(littleEndian); - - // char type = header[loc1+1]; - // assert(type == map_type(T)); - - std::string str_ws = header.substr(loc1 + 2); - loc2 = str_ws.find("'"); - word_size = atoi(str_ws.substr(0, loc2).c_str()); -} - -void cnpy::parse_zip_footer(FILE* fp, uint16_t& nrecs, - size_t& global_header_size, - size_t& global_header_offset) { - std::vector footer(22); - fseek(fp, -22, SEEK_END); - size_t res = fread(&footer[0], sizeof(char), 22, fp); - if (res != 22) - throw std::runtime_error("parse_zip_footer: failed fread"); - - uint16_t disk_no, disk_start, nrecs_on_disk, comment_len; - disk_no = *(uint16_t*) &footer[4]; - disk_start = *(uint16_t*) &footer[6]; - nrecs_on_disk = *(uint16_t*) &footer[8]; - nrecs = *(uint16_t*) &footer[10]; - global_header_size = *(uint32_t*) &footer[12]; - global_header_offset = *(uint32_t*) &footer[16]; - comment_len = *(uint16_t*) &footer[20]; - - assert(disk_no == 0); - assert(disk_start == 0); - assert(nrecs_on_disk == nrecs); - assert(comment_len == 0); -} - -cnpy::NpyArray load_the_npy_file(FILE* fp) { - std::vector shape; - size_t word_size; - bool fortran_order; - cnpy::parse_npy_header(fp, word_size, shape, fortran_order); - - cnpy::NpyArray arr(shape, word_size, fortran_order); - size_t nread = fread(arr.data(), 1, arr.num_bytes(), fp); - if (nread != arr.num_bytes()) - throw std::runtime_error("load_the_npy_file: failed fread"); - return arr; -} - -cnpy::NpyArray load_the_npz_array(FILE* fp, uint32_t compr_bytes, - uint32_t uncompr_bytes) { - - std::vector buffer_compr(compr_bytes); - std::vector buffer_uncompr(uncompr_bytes); - size_t nread = fread(&buffer_compr[0], 1, compr_bytes, fp); - if (nread != compr_bytes) - throw std::runtime_error("load_the_npy_file: failed fread"); - - int err; - z_stream d_stream; - - d_stream.zalloc = Z_NULL; - d_stream.zfree = Z_NULL; - d_stream.opaque = Z_NULL; - d_stream.avail_in = 0; - d_stream.next_in = Z_NULL; - err = inflateInit2(&d_stream, -MAX_WBITS); - - d_stream.avail_in = compr_bytes; - d_stream.next_in = &buffer_compr[0]; - d_stream.avail_out = uncompr_bytes; - d_stream.next_out = &buffer_uncompr[0]; - - err = inflate(&d_stream, Z_FINISH); - err = inflateEnd(&d_stream); - - std::vector shape; - size_t word_size; - bool fortran_order; - cnpy::parse_npy_header(&buffer_uncompr[0], word_size, shape, fortran_order); - - cnpy::NpyArray array(shape, word_size, fortran_order); - - size_t offset = uncompr_bytes - array.num_bytes(); - memcpy(array.data(), &buffer_uncompr[0] + offset, - array.num_bytes()); - - return array; -} - -cnpy::npz_t cnpy::npz_load(std::string fname) { - FILE* fp = fopen(fname.c_str(), "rb"); - - if (!fp) { - throw std::runtime_error("npz_load: Error! Unable to open file " + fname + - "!"); - } - - cnpy::npz_t arrays; - - while (1) { - std::vector local_header(30); - size_t headerres = fread(&local_header[0], sizeof(char), 30, fp); - if (headerres != 30) - throw std::runtime_error("npz_load: failed fread"); - - // if we've reached the global header, stop reading - if (local_header[2] != 0x03 || local_header[3] != 0x04) - break; - - // read in the variable name - uint16_t name_len = *(uint16_t*) &local_header[26]; - std::string varname(name_len, ' '); - size_t vname_res = fread(&varname[0], sizeof(char), name_len, fp); - if (vname_res != name_len) - throw std::runtime_error("npz_load: failed fread"); - - // erase the lagging .npy - varname.erase(varname.end() - 4, varname.end()); - - // read in the extra field - uint16_t extra_field_len = *(uint16_t*) &local_header[28]; - if (extra_field_len > 0) { - std::vector buff(extra_field_len); - size_t efield_res = fread(&buff[0], sizeof(char), extra_field_len, fp); - if (efield_res != extra_field_len) - throw std::runtime_error("npz_load: failed fread"); - } - - uint16_t compr_method = *reinterpret_cast(&local_header[0] + 8); - uint32_t compr_bytes = *reinterpret_cast(&local_header[0] + 18); - uint32_t uncompr_bytes = - *reinterpret_cast(&local_header[0] + 22); - - if (compr_method == 0) { - arrays[varname] = load_the_npy_file(fp); - } else { - arrays[varname] = load_the_npz_array(fp, compr_bytes, uncompr_bytes); - } - } - - fclose(fp); - return arrays; -} - -cnpy::NpyArray cnpy::npz_load(std::string fname, std::string varname) { - FILE* fp = fopen(fname.c_str(), "rb"); - - if (!fp) - throw std::runtime_error("npz_load: Unable to open file " + fname); - - while (1) { - std::vector local_header(30); - size_t header_res = fread(&local_header[0], sizeof(char), 30, fp); - if (header_res != 30) - throw std::runtime_error("npz_load: failed fread"); - - // if we've reached the global header, stop reading - if (local_header[2] != 0x03 || local_header[3] != 0x04) - break; - - // read in the variable name - uint16_t name_len = *(uint16_t*) &local_header[26]; - std::string vname(name_len, ' '); - size_t vname_res = fread(&vname[0], sizeof(char), name_len, fp); - if (vname_res != name_len) - throw std::runtime_error("npz_load: failed fread"); - vname.erase(vname.end() - 4, vname.end()); // erase the lagging .npy - - // read in the extra field - uint16_t extra_field_len = *(uint16_t*) &local_header[28]; - fseek(fp, extra_field_len, SEEK_CUR); // skip past the extra field - - uint16_t compr_method = *reinterpret_cast(&local_header[0] + 8); - uint32_t compr_bytes = *reinterpret_cast(&local_header[0] + 18); - uint32_t uncompr_bytes = - *reinterpret_cast(&local_header[0] + 22); - - if (vname == varname) { - NpyArray array = (compr_method == 0) - ? load_the_npy_file(fp) - : load_the_npz_array(fp, compr_bytes, uncompr_bytes); - fclose(fp); - return array; - } else { - // skip past the data - uint32_t size = *(uint32_t*) &local_header[22]; - fseek(fp, size, SEEK_CUR); - } - } - - fclose(fp); - - // if we get here, we haven't found the variable in the file - throw std::runtime_error("npz_load: Variable name " + varname + - " not found in " + fname); -} - -cnpy::NpyArray cnpy::npy_load(std::string fname) { - - FILE* fp = fopen(fname.c_str(), "rb"); - - if (!fp) - throw std::runtime_error("npy_load: Unable to open file " + fname); - - NpyArray arr = load_the_npy_file(fp); - - fclose(fp); - return arr; -} diff --git a/third_party/cnpy/cnpy.h b/third_party/cnpy/cnpy.h deleted file mode 100644 index 9361e60..0000000 --- a/third_party/cnpy/cnpy.h +++ /dev/null @@ -1,310 +0,0 @@ -// SPDX-FileCopyrightText: 2011 Carl Rogers -// -// SPDX-License-Identifier: MIT - -// Copyright (C) 2011 Carl Rogers -// Released under MIT License -// license available in LICENSE file, or at -// http://www.opensource.org/licenses/mit-license.php - -#ifndef LIBCNPY_H_ -#define LIBCNPY_H_ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace cnpy { - -struct NpyArray { - NpyArray(const std::vector& _shape, size_t _word_size, - bool _fortran_order) - : shape(_shape), word_size(_word_size), fortran_order(_fortran_order) { - num_vals = 1; - for (size_t i = 0; i < shape.size(); i++) - num_vals *= shape[i]; - data_holder = std::shared_ptr>( - new std::vector(num_vals * word_size)); - } - - NpyArray() : shape(0), word_size(0), fortran_order(0), num_vals(0) {} - - template - T* data() { - return reinterpret_cast(&(*data_holder)[0]); - } - - template - const T* data() const { - return reinterpret_cast(&(*data_holder)[0]); - } - - template - std::vector as_vec() const { - const T* p = data(); - return std::vector(p, p + num_vals); - } - - size_t num_bytes() const { - return data_holder->size(); - } - - std::shared_ptr> data_holder; - std::vector shape; - size_t word_size; - bool fortran_order; - size_t num_vals; -}; - -using npz_t = std::map; - -char BigEndianTest(); -char map_type(const std::type_info& t); -template -std::vector create_npy_header(const std::vector& shape); -void parse_npy_header(FILE* fp, size_t& word_size, std::vector& shape, - bool& fortran_order); -void parse_npy_header(unsigned char* buffer, size_t& word_size, - std::vector& shape, bool& fortran_order); -void parse_zip_footer(FILE* fp, uint16_t& nrecs, size_t& global_header_size, - size_t& global_header_offset); -npz_t npz_load(std::string fname); -NpyArray npz_load(std::string fname, std::string varname); -NpyArray npy_load(std::string fname); - -template -std::vector& operator+=(std::vector& lhs, const T rhs) { - // write in little endian - for (size_t byte = 0; byte < sizeof(T); byte++) { - char val = *((char*) &rhs + byte); - lhs.push_back(val); - } - return lhs; -} - -template <> -std::vector& operator+=(std::vector& lhs, const std::string rhs); -template <> -std::vector& operator+=(std::vector& lhs, const char* rhs); - -template -void npy_save(std::string fname, const T* data, const std::vector shape, - std::string mode = "w") { - FILE* fp = NULL; - std::vector - true_data_shape; // if appending, the shape of existing + new data - - if (mode == "a") - fp = fopen(fname.c_str(), "r+b"); - - if (fp) { - // file exists. we need to append to it. read the header, modify the array - // size - size_t word_size; - bool fortran_order; - parse_npy_header(fp, word_size, true_data_shape, fortran_order); - assert(!fortran_order); - - if (word_size != sizeof(T)) { - std::cout << "libnpy error: " << fname << " has word size " << word_size - << " but npy_save appending data sized " << sizeof(T) << "\n"; - assert(word_size == sizeof(T)); - } - if (true_data_shape.size() != shape.size()) { - std::cout << "libnpy error: npy_save attempting to append misdimensioned " - "data to " - << fname << "\n"; - assert(true_data_shape.size() != shape.size()); - } - - for (size_t i = 1; i < shape.size(); i++) { - if (shape[i] != true_data_shape[i]) { - std::cout - << "libnpy error: npy_save attempting to append misshaped data to " - << fname << "\n"; - assert(shape[i] == true_data_shape[i]); - } - } - true_data_shape[0] += shape[0]; - } else { - fp = fopen(fname.c_str(), "wb"); - true_data_shape = shape; - } - - std::vector header = create_npy_header(true_data_shape); - size_t nels = - std::accumulate(shape.begin(), shape.end(), 1, std::multiplies()); - - fseek(fp, 0, SEEK_SET); - fwrite(&header[0], sizeof(char), header.size(), fp); - fseek(fp, 0, SEEK_END); - fwrite(data, sizeof(T), nels, fp); - fclose(fp); -} - -template -void npz_save(std::string zipname, std::string fname, const T* data, - const std::vector& shape, std::string mode = "w") { - // first, append a .npy to the fname - fname += ".npy"; - - // now, on with the show - FILE* fp = NULL; - uint16_t nrecs = 0; - size_t global_header_offset = 0; - std::vector global_header; - - if (mode == "a") - fp = fopen(zipname.c_str(), "r+b"); - - if (fp) { - // zip file exists. we need to add a new npy file to it. - // first read the footer. this gives us the offset and size of the global - // header then read and store the global header. below, we will write the - // the new data at the start of the global header then append the global - // header and footer below it - size_t global_header_size; - parse_zip_footer(fp, nrecs, global_header_size, global_header_offset); - fseek(fp, global_header_offset, SEEK_SET); - global_header.resize(global_header_size); - size_t res = fread(&global_header[0], sizeof(char), global_header_size, fp); - if (res != global_header_size) { - throw std::runtime_error( - "npz_save: header read error while adding to existing zip"); - } - fseek(fp, global_header_offset, SEEK_SET); - } else { - fp = fopen(zipname.c_str(), "wb"); - } - - std::vector npy_header = create_npy_header(shape); - - size_t nels = - std::accumulate(shape.begin(), shape.end(), 1, std::multiplies()); - size_t nbytes = nels * sizeof(T) + npy_header.size(); - - // get the CRC of the data to be added - uint32_t crc = crc32(0L, (uint8_t*) &npy_header[0], npy_header.size()); - crc = crc32(crc, (uint8_t*) data, nels * sizeof(T)); - - // build the local header - std::vector local_header; - local_header += "PK"; // first part of sig - local_header += (uint16_t) 0x0403; // second part of sig - local_header += (uint16_t) 20; // min version to extract - local_header += (uint16_t) 0; // general purpose bit flag - local_header += (uint16_t) 0; // compression method - local_header += (uint16_t) 0; // file last mod time - local_header += (uint16_t) 0; // file last mod date - local_header += (uint32_t) crc; // crc - local_header += (uint32_t) nbytes; // compressed size - local_header += (uint32_t) nbytes; // uncompressed size - local_header += (uint16_t) fname.size(); // fname length - local_header += (uint16_t) 0; // extra field length - local_header += fname; - - // build global header - global_header += "PK"; // first part of sig - global_header += (uint16_t) 0x0201; // second part of sig - global_header += (uint16_t) 20; // version made by - global_header.insert(global_header.end(), local_header.begin() + 4, - local_header.begin() + 30); - global_header += (uint16_t) 0; // file comment length - global_header += (uint16_t) 0; // disk number where file starts - global_header += (uint16_t) 0; // internal file attributes - global_header += (uint32_t) 0; // external file attributes - global_header += (uint32_t) - global_header_offset; // relative offset of local file header, since it - // begins where the global header used to begin - global_header += fname; - - // build footer - std::vector footer; - footer += "PK"; // first part of sig - footer += (uint16_t) 0x0605; // second part of sig - footer += (uint16_t) 0; // number of this disk - footer += (uint16_t) 0; // disk where footer starts - footer += (uint16_t) (nrecs + 1); // number of records on this disk - footer += (uint16_t) (nrecs + 1); // total number of records - footer += (uint32_t) global_header.size(); // nbytes of global headers - footer += - (uint32_t) (global_header_offset + nbytes + - local_header.size()); // offset of start of global headers, - // since global header now starts after - // newly written array - footer += (uint16_t) 0; // zip file comment length - - // write everything - fwrite(&local_header[0], sizeof(char), local_header.size(), fp); - fwrite(&npy_header[0], sizeof(char), npy_header.size(), fp); - fwrite(data, sizeof(T), nels, fp); - fwrite(&global_header[0], sizeof(char), global_header.size(), fp); - fwrite(&footer[0], sizeof(char), footer.size(), fp); - fclose(fp); -} - -template -void npy_save(std::string fname, const std::vector data, - std::string mode = "w") { - std::vector shape; - shape.push_back(data.size()); - npy_save(fname, &data[0], shape, mode); -} - -template -void npz_save(std::string zipname, std::string fname, const std::vector data, - std::string mode = "w") { - std::vector shape; - shape.push_back(data.size()); - npz_save(zipname, fname, &data[0], shape, mode); -} - -template -std::vector create_npy_header(const std::vector& shape) { - - std::vector dict; - dict += "{'descr': '"; - dict += BigEndianTest(); - dict += map_type(typeid(T)); - dict += std::to_string(sizeof(T)); - dict += "', 'fortran_order': False, 'shape': ("; - if (!shape.empty()) { - dict += std::to_string(shape[0]); - for (size_t i = 1; i < shape.size(); i++) { - dict += ", "; - dict += std::to_string(shape[i]); - } - } - if (shape.size() == 1) - dict += ","; - dict += "), }"; - // pad with spaces so that preamble+dict is modulo 16 bytes. preamble is 10 - // bytes. dict needs to end with \n - int remainder = 16 - (10 + dict.size()) % 16; - dict.insert(dict.end(), remainder, ' '); - dict.back() = '\n'; - - std::vector header; - header += (char) 0x93; - header += "NUMPY"; - header += (char) 0x01; // major version of numpy format - header += (char) 0x00; // minor version of numpy format - header += (uint16_t) dict.size(); - header.insert(header.end(), dict.begin(), dict.end()); - - return header; -} - -} // namespace cnpy - -#endif diff --git a/third_party/cnpy/example1.cpp b/third_party/cnpy/example1.cpp deleted file mode 100644 index 3ea5536..0000000 --- a/third_party/cnpy/example1.cpp +++ /dev/null @@ -1,64 +0,0 @@ -// SPDX-FileCopyrightText: 2011 Carl Rogers -// -// SPDX-License-Identifier: MIT - -#include "cnpy.h" -#include -#include -#include -#include -#include - -const int Nx = 128; -const int Ny = 64; -const int Nz = 32; - -int main() { - // set random seed so that result is reproducible (for testing) - srand(0); - // create random data - std::vector> data(Nx * Ny * Nz); - for (int i = 0; i < Nx * Ny * Nz; i++) - data[i] = std::complex(rand(), rand()); - - // save it to file - cnpy::npy_save("arr1.npy", &data[0], {Nz, Ny, Nx}, "w"); - - // load it into a new array - cnpy::NpyArray arr = cnpy::npy_load("arr1.npy"); - std::complex* loaded_data = arr.data>(); - - // make sure the loaded data matches the saved data - assert(arr.word_size == sizeof(std::complex)); - assert(arr.shape.size() == 3 && arr.shape[0] == Nz && arr.shape[1] == Ny && - arr.shape[2] == Nx); - for (int i = 0; i < Nx * Ny * Nz; i++) - assert(data[i] == loaded_data[i]); - - // append the same data to file - // npy array on file now has shape (Nz+Nz,Ny,Nx) - cnpy::npy_save("arr1.npy", &data[0], {Nz, Ny, Nx}, "a"); - - // now write to an npz file - // non-array variables are treated as 1D arrays with 1 element - double myVar1 = 1.2; - char myVar2 = 'a'; - cnpy::npz_save("out.npz", "myVar1", &myVar1, {1}, - "w"); //"w" overwrites any existing file - cnpy::npz_save("out.npz", "myVar2", &myVar2, {1}, - "a"); //"a" appends to the file we created above - cnpy::npz_save("out.npz", "arr1", &data[0], {Nz, Ny, Nx}, - "a"); //"a" appends to the file we created above - - // load a single var from the npz file - cnpy::NpyArray arr2 = cnpy::npz_load("out.npz", "arr1"); - - // load the entire npz file - cnpy::npz_t my_npz = cnpy::npz_load("out.npz"); - - // check that the loaded myVar1 matches myVar1 - cnpy::NpyArray arr_mv1 = my_npz["myVar1"]; - double* mv1 = arr_mv1.data(); - assert(arr_mv1.shape.size() == 1 && arr_mv1.shape[0] == 1); - assert(mv1[0] == myVar1); -} diff --git a/third_party/cnpy/mat2npz b/third_party/cnpy/mat2npz deleted file mode 100644 index 6259342..0000000 --- a/third_party/cnpy/mat2npz +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env python - -# SPDX-FileCopyrightText: 2011 Carl Rogers -# -# SPDX-License-Identifier: MIT - -import sys -from numpy import savez -from scipy.io import loadmat - -assert len(sys.argv) > 1 - -files = sys.argv[1:] - -for f in files: - mat_vars = loadmat(f) - mat_vars.pop('__version__') - mat_vars.pop('__header__') - mat_vars.pop('__globals__') - - fn = f.replace('.mat','.npz') - savez(fn,**mat_vars) diff --git a/third_party/cnpy/npy2mat b/third_party/cnpy/npy2mat deleted file mode 100644 index 1eeecc7..0000000 --- a/third_party/cnpy/npy2mat +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env python - -# SPDX-FileCopyrightText: 2011 Carl Rogers -# -# SPDX-License-Identifier: MIT - -import sys -from numpy import load -from scipy.io import savemat - -assert len(sys.argv) > 1 - -files = sys.argv[1:] - -for f in files: - data = load(f) - fn = f.replace('.npy','') - fn = fn.replace('.','_') - savemat(fn,{fn : data}) diff --git a/third_party/cnpy/npz2mat b/third_party/cnpy/npz2mat deleted file mode 100755 index 14e89e6..0000000 --- a/third_party/cnpy/npz2mat +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env python - -# SPDX-FileCopyrightText: 2011 Carl Rogers -# -# SPDX-License-Identifier: MIT - -import sys -from numpy import load -from scipy.io import savemat - -assert len(sys.argv) > 1 - -files = sys.argv[1:] - -for f in files: - data = load(f) - fn = f.replace('.npz','') - fn = fn.replace('.','_') #matlab cant handle dots - savemat(fn,data) From d87eaa9a460183e11722073a608c0a2dfb50d68f Mon Sep 17 00:00:00 2001 From: Willow Ahrens Date: Tue, 1 Sep 2026 22:00:06 -0400 Subject: [PATCH 13/16] precommit --- LICENSES/MIT.txt | 21 -- include/binsparse/version.h | 2 +- src/write_tensor.c | 8 +- test/c/version_test.c | 14 +- test/compliance/CMakeLists.txt | 28 +- test/compliance/binsparse_to_npy.c | 8 +- test/compliance/cnpy_c.cpp | 115 -------- test/compliance/npy_c.c | 375 +++++++++++++++++++++++++ test/compliance/{cnpy_c.h => npy_c.h} | 10 +- test/compliance/npy_to_binsparse.c | 16 +- test/compliance/run-binsparse-tests.sh | 2 +- 11 files changed, 411 insertions(+), 188 deletions(-) delete mode 100644 LICENSES/MIT.txt delete mode 100644 test/compliance/cnpy_c.cpp create mode 100644 test/compliance/npy_c.c rename test/compliance/{cnpy_c.h => npy_c.h} (58%) diff --git a/LICENSES/MIT.txt b/LICENSES/MIT.txt deleted file mode 100644 index aa29325..0000000 --- a/LICENSES/MIT.txt +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2011 Carl Rogers - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/include/binsparse/version.h b/include/binsparse/version.h index 008a65f..5a47da7 100644 --- a/include/binsparse/version.h +++ b/include/binsparse/version.h @@ -8,7 +8,7 @@ #include -#define BINSPARSE_VERSION "0.1.0" +#define BINSPARSE_VERSION "0.1" #ifdef __cplusplus extern "C" { diff --git a/src/write_tensor.c b/src/write_tensor.c index 27751fe..60bb723 100644 --- a/src/write_tensor.c +++ b/src/write_tensor.c @@ -193,8 +193,8 @@ bsp_error_t bsp_write_tensor_cjson(const char* fname, bsp_tensor_t tensor, int compression_level) { if (group == NULL) { hid_t f = H5Fcreate(fname, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); - bsp_error_t error = - bsp_write_tensor_to_group_cjson(f, tensor, user_json, compression_level); + bsp_error_t error = bsp_write_tensor_to_group_cjson(f, tensor, user_json, + compression_level); if (error != BSP_SUCCESS) { H5Fclose(f); return error; @@ -208,8 +208,8 @@ bsp_error_t bsp_write_tensor_cjson(const char* fname, bsp_tensor_t tensor, f = H5Fcreate(fname, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); } hid_t g = H5Gcreate1(f, group, H5P_DEFAULT); - bsp_error_t error = - bsp_write_tensor_to_group_cjson(g, tensor, user_json, compression_level); + bsp_error_t error = bsp_write_tensor_to_group_cjson(g, tensor, user_json, + compression_level); if (error != BSP_SUCCESS) { H5Gclose(g); H5Fclose(f); diff --git a/test/c/version_test.c b/test/c/version_test.c index 8093638..5309a0e 100644 --- a/test/c/version_test.c +++ b/test/c/version_test.c @@ -28,12 +28,12 @@ static void write_versioned_dvec(const char* filename, const char* version) { bsp_destroy_matrix_t(&matrix); char json[1024]; - int length = snprintf( - json, sizeof(json), - "{\"binsparse\":{\"version\":\"%s\",\"format\":\"DVEC\"," - "\"shape\":[2],\"number_of_stored_values\":2," - "\"data_types\":{\"values\":\"float64\"}}}", - version); + int length = + snprintf(json, sizeof(json), + "{\"binsparse\":{\"version\":\"%s\",\"format\":\"DVEC\"," + "\"shape\":[2],\"number_of_stored_values\":2," + "\"data_types\":{\"values\":\"float64\"}}}", + version); assert(length > 0 && (size_t) length < sizeof(json)); hid_t file = H5Fopen(filename, H5F_ACC_RDWR, H5P_DEFAULT); @@ -61,6 +61,7 @@ int main(int argc, char** argv) { assert(argc == 2); const char* filename = argv[1]; + assert(bsp_check_version_compatible("0.1") == BSP_SUCCESS); assert(bsp_check_version_compatible("0.1.0") == BSP_SUCCESS); assert(bsp_check_version_compatible("0.1.0+roundtrip.1") == BSP_SUCCESS); assert(bsp_check_version_compatible("0.1.1") == BSP_ERROR_UNSUPPORTED); @@ -75,6 +76,7 @@ int main(int argc, char** argv) { assert(bsp_check_version_compatible(NULL) == BSP_ERROR_FORMAT); check_matrix_reader(filename, "0.1.0", BSP_SUCCESS); + check_matrix_reader(filename, "0.1", BSP_SUCCESS); check_matrix_reader(filename, "0.1.0+roundtrip.1", BSP_SUCCESS); check_matrix_reader(filename, "0.1.7", BSP_ERROR_UNSUPPORTED); check_matrix_reader(filename, "0.1.0-alpha.1", BSP_SUCCESS); diff --git a/test/compliance/CMakeLists.txt b/test/compliance/CMakeLists.txt index e59d498..1cc90a7 100644 --- a/test/compliance/CMakeLists.txt +++ b/test/compliance/CMakeLists.txt @@ -4,40 +4,22 @@ SPDX-FileCopyrightText: 2026 Binsparse Developers SPDX-License-Identifier: BSD-3-Clause ]] -find_package(ZLIB REQUIRED) -enable_language(CXX) -include(FetchContent) - -FetchContent_Declare( - cnpy - GIT_REPOSITORY https://github.com/rogersce/cnpy.git - GIT_TAG 4e8810b1a8637695171ed346ce68f6984e585ef4 - SOURCE_SUBDIR cmake -) -FetchContent_MakeAvailable(cnpy) - -add_library(cnpy_vendor STATIC - ${cnpy_SOURCE_DIR}/cnpy.cpp - cnpy_c.cpp +add_library(npy_bridge STATIC + npy_c.c ) -target_include_directories(cnpy_vendor PUBLIC - ${cnpy_SOURCE_DIR} +target_include_directories(npy_bridge PUBLIC ${CMAKE_CURRENT_SOURCE_DIR} ) -target_link_libraries(cnpy_vendor PUBLIC ZLIB::ZLIB) -target_compile_features(cnpy_vendor PUBLIC cxx_std_11) add_library(compliance_core STATIC reformat.c) target_include_directories(compliance_core PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) target_link_libraries(compliance_core PUBLIC binsparse) add_executable(npy_to_binsparse npy_to_binsparse.c) -target_link_libraries(npy_to_binsparse PRIVATE compliance_core cnpy_vendor) -set_target_properties(npy_to_binsparse PROPERTIES LINKER_LANGUAGE CXX) +target_link_libraries(npy_to_binsparse PRIVATE compliance_core npy_bridge) add_executable(binsparse_to_npy binsparse_to_npy.c) -target_link_libraries(binsparse_to_npy PRIVATE compliance_core cnpy_vendor) -set_target_properties(binsparse_to_npy PROPERTIES LINKER_LANGUAGE CXX) +target_link_libraries(binsparse_to_npy PRIVATE compliance_core npy_bridge) add_executable(binsparse_to_binsparse binsparse_to_binsparse.c) target_link_libraries(binsparse_to_binsparse PRIVATE compliance_core) diff --git a/test/compliance/binsparse_to_npy.c b/test/compliance/binsparse_to_npy.c index d7af64c..7cd2439 100644 --- a/test/compliance/binsparse_to_npy.c +++ b/test/compliance/binsparse_to_npy.c @@ -4,7 +4,7 @@ * SPDX-License-Identifier: BSD-3-Clause */ -#include "cnpy_c.h" +#include "npy_c.h" #include "reformat.h" #include @@ -121,9 +121,9 @@ int main(int argc, char** argv) { entries.entry[k].value * bsp_type_size(values.type), bsp_type_size(values.type)); } - if (cnpy_c_save(argv[2], dense, shape, (size_t) rank, values.type) || - cnpy_c_save(argv[3], pattern, shape, (size_t) rank, BSP_BINT8) || - cnpy_c_save(argv[4], fill_value.data, NULL, 0, fill_value.type)) + if (npy_c_save(argv[2], dense, shape, (size_t) rank, values.type) || + npy_c_save(argv[3], pattern, shape, (size_t) rank, BSP_BINT8) || + npy_c_save(argv[4], fill_value.data, NULL, 0, fill_value.type)) die("cannot write NPY output"); free(dense); diff --git a/test/compliance/cnpy_c.cpp b/test/compliance/cnpy_c.cpp deleted file mode 100644 index 273fc04..0000000 --- a/test/compliance/cnpy_c.cpp +++ /dev/null @@ -1,115 +0,0 @@ -// SPDX-FileCopyrightText: 2026 Binsparse Developers -// -// SPDX-License-Identifier: BSD-3-Clause - -#include "cnpy_c.h" - -#include - -#include -#include -#include -#include -#include - -extern "C" int cnpy_c_load(const char* path, cnpy_c_array* out) { - try { - cnpy::NpyArray array = cnpy::npy_load(path); - out->size = array.num_vals; - out->word_size = array.word_size; - out->rank = array.shape.size(); - out->shape = static_cast(malloc(out->rank * sizeof(size_t))); - out->strides = static_cast(malloc(out->rank * sizeof(size_t))); - out->data = malloc(array.num_bytes()); - if ((out->rank && (!out->shape || !out->strides)) || - (array.num_bytes() && !out->data)) - return 1; - if (out->rank) - memcpy(out->shape, array.shape.data(), out->rank * sizeof(size_t)); - size_t stride = 1; - if (array.fortran_order) { - for (size_t d = 0; d < out->rank; ++d) { - out->strides[d] = stride; - stride *= out->shape[d]; - } - } else { - for (size_t d = out->rank; d-- > 0;) { - out->strides[d] = stride; - stride *= out->shape[d]; - } - } - if (array.num_bytes()) - memcpy(out->data, array.data(), array.num_bytes()); - return 0; - } catch (const std::exception&) { - return 1; - } -} - -extern "C" void cnpy_c_destroy(cnpy_c_array* array) { - free(array->data); - free(array->shape); - free(array->strides); - memset(array, 0, sizeof(*array)); -} - -template -static void save(const char* path, const void* data, const size_t* shape, - size_t rank) { - std::vector dimensions; - if (rank != 0) - dimensions.assign(shape, shape + rank); - cnpy::npy_save(path, static_cast(data), dimensions); -} - -extern "C" int cnpy_c_save(const char* path, const void* data, - const size_t* shape, size_t rank, int type) { - try { - switch (type) { - case 0: - save(path, data, shape, rank); - break; - case 1: - save(path, data, shape, rank); - break; - case 2: - save(path, data, shape, rank); - break; - case 3: - save(path, data, shape, rank); - break; - case 4: - save(path, data, shape, rank); - break; - case 5: - save(path, data, shape, rank); - break; - case 6: - save(path, data, shape, rank); - break; - case 7: - save(path, data, shape, rank); - break; - case 8: - save(path, data, shape, rank); - break; - case 9: - save(path, data, shape, rank); - break; - case 10: - save(path, data, shape, rank); - break; - case 11: - save>(path, data, shape, rank); - break; - case 12: - save>(path, data, shape, rank); - break; - default: - return 1; - } - return 0; - } catch (const std::exception&) { - return 1; - } -} diff --git a/test/compliance/npy_c.c b/test/compliance/npy_c.c new file mode 100644 index 0000000..6565d2e --- /dev/null +++ b/test/compliance/npy_c.c @@ -0,0 +1,375 @@ +/* + * SPDX-FileCopyrightText: 2026 Binsparse Developers + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include "npy_c.h" + +#include +#include +#include +#include +#include +#include + +typedef struct { + size_t word_size; + bool fortran_order; + size_t rank; + size_t* shape; +} npy_header_t; + +static const char* skip_spaces(const char* text) { + while (*text == ' ' || *text == '\t' || *text == '\n' || *text == '\r') + ++text; + return text; +} + +static const char* value_for_key(const char* header, const char* key) { + const char* key_start = strstr(header, key); + if (key_start == NULL) + return NULL; + const char* value = skip_spaces(key_start + strlen(key)); + if (*value != ':') + return NULL; + return skip_spaces(value + 1); +} + +static int parse_descr(const char* header, char* kind, size_t* word_size) { + const char* value = value_for_key(header, "'descr'"); + if (value == NULL) + value = value_for_key(header, "\"descr\""); + if (value == NULL || (*value != '\'' && *value != '"')) + return 1; + + char quote = *value++; + char endian = *value++; + if (endian != '<' && endian != '>' && endian != '|' && endian != '=') + return 1; + *kind = *value++; + if (*kind != 'b' && *kind != 'i' && *kind != 'u' && *kind != 'f' && + *kind != 'c') + return 1; + + char* end = NULL; + unsigned long parsed = strtoul(value, &end, 10); + if (end == value || *end != quote || parsed == 0) + return 1; + *word_size = (size_t) parsed; + return 0; +} + +static int parse_fortran_order(const char* header, bool* fortran_order) { + const char* value = value_for_key(header, "'fortran_order'"); + if (value == NULL) + value = value_for_key(header, "\"fortran_order\""); + if (value == NULL) + return 1; + if (!strncmp(value, "True", 4)) { + *fortran_order = true; + return 0; + } + if (!strncmp(value, "False", 5)) { + *fortran_order = false; + return 0; + } + return 1; +} + +static int append_shape_dim(size_t** shape, size_t* rank, size_t dim) { + size_t* resized = realloc(*shape, (*rank + 1) * sizeof(size_t)); + if (resized == NULL) + return 1; + resized[*rank] = dim; + *shape = resized; + ++*rank; + return 0; +} + +static int parse_shape(const char* header, size_t** shape, size_t* rank) { + const char* value = value_for_key(header, "'shape'"); + if (value == NULL) + value = value_for_key(header, "\"shape\""); + if (value == NULL) + return 1; + value = skip_spaces(value); + if (*value != '(') + return 1; + ++value; + + *shape = NULL; + *rank = 0; + for (;;) { + value = skip_spaces(value); + if (*value == ')') + return 0; + + char* end = NULL; + unsigned long parsed = strtoul(value, &end, 10); + if (end == value) + return 1; + if (append_shape_dim(shape, rank, (size_t) parsed)) + return 1; + + value = skip_spaces(end); + if (*value == ',') { + ++value; + continue; + } + if (*value == ')') + return 0; + return 1; + } +} + +static void destroy_header(npy_header_t* header) { + free(header->shape); + memset(header, 0, sizeof(*header)); +} + +static uint16_t read_le16(const uint8_t bytes[2]) { + return (uint16_t) bytes[0] | ((uint16_t) bytes[1] << 8); +} + +static uint32_t read_le32(const uint8_t bytes[4]) { + return (uint32_t) bytes[0] | ((uint32_t) bytes[1] << 8) | + ((uint32_t) bytes[2] << 16) | ((uint32_t) bytes[3] << 24); +} + +static int read_header(FILE* file, npy_header_t* out) { + uint8_t prefix[10]; + if (fread(prefix, 1, sizeof(prefix), file) != sizeof(prefix)) + return 1; + const uint8_t magic[] = {0x93, 'N', 'U', 'M', 'P', 'Y'}; + if (memcmp(prefix, magic, sizeof(magic))) + return 1; + + uint8_t major = prefix[6]; + uint8_t minor = prefix[7]; + size_t header_size; + if (major == 1 && minor == 0) { + header_size = read_le16(prefix + 8); + } else if ((major == 2 || major == 3) && minor == 0) { + uint8_t length[4]; + length[0] = prefix[8]; + length[1] = prefix[9]; + if (fread(length + 2, 1, 2, file) != 2) + return 1; + header_size = read_le32(length); + } else { + return 1; + } + + char* header = malloc(header_size + 1); + if (header == NULL) + return 1; + if (fread(header, 1, header_size, file) != header_size) { + free(header); + return 1; + } + header[header_size] = '\0'; + + char kind; + int error = parse_descr(header, &kind, &out->word_size) || + parse_fortran_order(header, &out->fortran_order) || + parse_shape(header, &out->shape, &out->rank); + free(header); + return error; +} + +static int checked_mul(size_t lhs, size_t rhs, size_t* result) { + if (rhs != 0 && lhs > (size_t) -1 / rhs) + return 1; + *result = lhs * rhs; + return 0; +} + +static int element_count(const size_t* shape, size_t rank, size_t* count) { + *count = 1; + for (size_t i = 0; i < rank; ++i) + if (checked_mul(*count, shape[i], count)) + return 1; + return 0; +} + +extern int npy_c_load(const char* path, npy_c_array* out) { + FILE* file = fopen(path, "rb"); + if (file == NULL) + return 1; + + npy_header_t header = {0}; + if (read_header(file, &header)) { + fclose(file); + destroy_header(&header); + return 1; + } + + size_t count; + size_t bytes; + if (element_count(header.shape, header.rank, &count) || + checked_mul(count, header.word_size, &bytes)) { + fclose(file); + destroy_header(&header); + return 1; + } + + out->size = count; + out->word_size = header.word_size; + out->rank = header.rank; + out->shape = header.rank ? malloc(header.rank * sizeof(size_t)) : NULL; + out->strides = header.rank ? malloc(header.rank * sizeof(size_t)) : NULL; + out->data = bytes ? malloc(bytes) : NULL; + if ((header.rank && (!out->shape || !out->strides)) || + (bytes && out->data == NULL)) { + fclose(file); + destroy_header(&header); + npy_c_destroy(out); + return 1; + } + if (header.rank) + memcpy(out->shape, header.shape, header.rank * sizeof(size_t)); + + size_t stride = 1; + if (header.fortran_order) { + for (size_t d = 0; d < out->rank; ++d) { + out->strides[d] = stride; + stride *= out->shape[d]; + } + } else { + for (size_t d = out->rank; d-- > 0;) { + out->strides[d] = stride; + stride *= out->shape[d]; + } + } + + int error = bytes && fread(out->data, 1, bytes, file) != bytes; + fclose(file); + destroy_header(&header); + return error; +} + +extern void npy_c_destroy(npy_c_array* array) { + free(array->data); + free(array->shape); + free(array->strides); + memset(array, 0, sizeof(*array)); +} + +static char endian_char(void) { + const uint16_t value = 1; + return *(const uint8_t*) &value ? '<' : '>'; +} + +static int write_shape(char* buffer, size_t size, const size_t* shape, + size_t rank) { + size_t used = 0; + int written = snprintf(buffer, size, "("); + if (written < 0 || (size_t) written >= size) + return -1; + used += (size_t) written; + + for (size_t i = 0; i < rank; ++i) { + written = snprintf(buffer + used, size - used, "%s%zu", i == 0 ? "" : ", ", + shape[i]); + if (written < 0 || (size_t) written >= size - used) + return -1; + used += (size_t) written; + } + if (rank == 1) { + written = snprintf(buffer + used, size - used, ","); + if (written < 0 || (size_t) written >= size - used) + return -1; + used += (size_t) written; + } + written = snprintf(buffer + used, size - used, ")"); + if (written < 0 || (size_t) written >= size - used) + return -1; + return 0; +} + +static int write_npy(const char* path, const void* data, const size_t* shape, + size_t rank, size_t word_size, char kind) { + FILE* file = fopen(path, "wb"); + if (file == NULL) + return 1; + + char shape_text[512]; + if (write_shape(shape_text, sizeof(shape_text), shape, rank)) { + fclose(file); + return 1; + } + + char header[1024]; + char endian = word_size == 1 ? '|' : endian_char(); + int header_length = + snprintf(header, sizeof(header), + "{'descr': '%c%c%zu', 'fortran_order': False, 'shape': %s, }", + endian, kind, word_size, shape_text); + if (header_length < 0 || (size_t) header_length >= sizeof(header)) { + fclose(file); + return 1; + } + + size_t header_size = (size_t) header_length; + size_t padding = 16 - (10 + header_size) % 16; + if (header_size + padding >= sizeof(header)) { + fclose(file); + return 1; + } + memset(header + header_size, ' ', padding); + header_size += padding; + header[header_size - 1] = '\n'; + + const uint8_t magic[] = {0x93, 'N', 'U', 'M', 'P', 'Y', 0x01, 0x00}; + uint8_t header_size_bytes[] = { + (uint8_t) (header_size & 0xff), + (uint8_t) ((header_size >> 8) & 0xff), + }; + size_t count; + if (element_count(shape, rank, &count)) { + fclose(file); + return 1; + } + + int ok = fwrite(magic, 1, sizeof(magic), file) == sizeof(magic) && + fwrite(header_size_bytes, 1, sizeof(header_size_bytes), file) == + sizeof(header_size_bytes) && + fwrite(header, 1, header_size, file) == header_size && + (count == 0 || fwrite(data, word_size, count, file) == count); + return fclose(file) == 0 && ok ? 0 : 1; +} + +extern int npy_c_save(const char* path, const void* data, const size_t* shape, + size_t rank, int type) { + switch (type) { + case 0: + return write_npy(path, data, shape, rank, sizeof(uint8_t), 'u'); + case 1: + return write_npy(path, data, shape, rank, sizeof(uint16_t), 'u'); + case 2: + return write_npy(path, data, shape, rank, sizeof(uint32_t), 'u'); + case 3: + return write_npy(path, data, shape, rank, sizeof(uint64_t), 'u'); + case 4: + return write_npy(path, data, shape, rank, sizeof(int8_t), 'i'); + case 5: + return write_npy(path, data, shape, rank, sizeof(int16_t), 'i'); + case 6: + return write_npy(path, data, shape, rank, sizeof(int32_t), 'i'); + case 7: + return write_npy(path, data, shape, rank, sizeof(int64_t), 'i'); + case 8: + return write_npy(path, data, shape, rank, sizeof(float), 'f'); + case 9: + return write_npy(path, data, shape, rank, sizeof(double), 'f'); + case 10: + return write_npy(path, data, shape, rank, sizeof(bool), 'b'); + case 11: + return write_npy(path, data, shape, rank, sizeof(float _Complex), 'c'); + case 12: + return write_npy(path, data, shape, rank, sizeof(double _Complex), 'c'); + default: + return 1; + } +} diff --git a/test/compliance/cnpy_c.h b/test/compliance/npy_c.h similarity index 58% rename from test/compliance/cnpy_c.h rename to test/compliance/npy_c.h index 13af018..930278b 100644 --- a/test/compliance/cnpy_c.h +++ b/test/compliance/npy_c.h @@ -19,12 +19,12 @@ typedef struct { size_t rank; size_t* shape; size_t* strides; -} cnpy_c_array; +} npy_c_array; -int cnpy_c_load(const char* path, cnpy_c_array* array); -void cnpy_c_destroy(cnpy_c_array* array); -int cnpy_c_save(const char* path, const void* data, const size_t* shape, - size_t rank, int bsp_type); +int npy_c_load(const char* path, npy_c_array* array); +void npy_c_destroy(npy_c_array* array); +int npy_c_save(const char* path, const void* data, const size_t* shape, + size_t rank, int bsp_type); #ifdef __cplusplus } diff --git a/test/compliance/npy_to_binsparse.c b/test/compliance/npy_to_binsparse.c index f5f9b50..db48390 100644 --- a/test/compliance/npy_to_binsparse.c +++ b/test/compliance/npy_to_binsparse.c @@ -4,7 +4,7 @@ * SPDX-License-Identifier: BSD-3-Clause */ -#include "cnpy_c.h" +#include "npy_c.h" #include "reformat.h" #include @@ -21,7 +21,7 @@ static size_t product(const size_t* shape, size_t rank) { return size; } -static size_t offset(const cnpy_c_array* array, const size_t* coord) { +static size_t offset(const npy_c_array* array, const size_t* coord) { size_t result = 0; for (size_t d = 0; d < array->rank; ++d) result += coord[d] * array->strides[d]; @@ -29,7 +29,7 @@ static size_t offset(const cnpy_c_array* array, const size_t* coord) { } int main(int argc, char** argv) { - cnpy_c_array dense = {0}, pattern = {0}, fill = {0}; + npy_c_array dense = {0}, pattern = {0}, fill = {0}; cJSON *header, *shape_json, *custom, *description; const char *format, *value_name; bsp_type_t value_type; @@ -47,8 +47,8 @@ int main(int argc, char** argv) { " \n"); return 2; } - if (cnpy_c_load(argv[1], &dense) || cnpy_c_load(argv[2], &pattern) || - cnpy_c_load(argv[3], &fill)) + if (npy_c_load(argv[1], &dense) || npy_c_load(argv[2], &pattern) || + npy_c_load(argv[3], &fill)) die("cannot read NPY input"); if (dense.rank != pattern.rank || dense.size != pattern.size || pattern.word_size != 1 || fill.size != 1) @@ -181,8 +181,8 @@ int main(int argc, char** argv) { free(stored_dims); cJSON_Delete(description); cJSON_Delete(header); - cnpy_c_destroy(&dense); - cnpy_c_destroy(&pattern); - cnpy_c_destroy(&fill); + npy_c_destroy(&dense); + npy_c_destroy(&pattern); + npy_c_destroy(&fill); return 0; } diff --git a/test/compliance/run-binsparse-tests.sh b/test/compliance/run-binsparse-tests.sh index c562653..a772a7b 100755 --- a/test/compliance/run-binsparse-tests.sh +++ b/test/compliance/run-binsparse-tests.sh @@ -9,7 +9,7 @@ script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" repo_root="$(cd -- "${script_dir}/../.." && pwd)" build_dir="${BINSPARSE_BUILD_DIR:-${repo_root}/build-compliance}" tests_dir="${BINSPARSE_TESTS_DIR:-${build_dir}/binsparse-tests}" -tests_ref="${BINSPARSE_TESTS_REF:-main}" +tests_ref="${BINSPARSE_TESTS_REF:-9df16f6e7fe6d73c147ea5a3c23cf203876386b6}" build_config="${BINSPARSE_BUILD_CONFIG:-Release}" for command in git pixi; do From 13e56902c6f729822730297a5d13bbdbc62e619b Mon Sep 17 00:00:00 2001 From: Willow Ahrens Date: Tue, 1 Sep 2026 22:05:26 -0400 Subject: [PATCH 14/16] Fix pinned compliance test checkout --- test/compliance/run-binsparse-tests.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/compliance/run-binsparse-tests.sh b/test/compliance/run-binsparse-tests.sh index a772a7b..23fe6dd 100755 --- a/test/compliance/run-binsparse-tests.sh +++ b/test/compliance/run-binsparse-tests.sh @@ -26,8 +26,10 @@ elif [[ -e "${tests_dir}" ]]; then echo "error: ${tests_dir} exists but is not a Git checkout" >&2 exit 1 else - git clone --depth 1 --branch "${tests_ref}" \ - https://github.com/Binsparse/binsparse-tests.git "${tests_dir}" + git clone --depth 1 https://github.com/Binsparse/binsparse-tests.git \ + "${tests_dir}" + git -C "${tests_dir}" fetch --depth 1 origin "${tests_ref}" + git -C "${tests_dir}" checkout --detach FETCH_HEAD fi ( From d11341915d03be4438a56ba2e9cb49d06219874b Mon Sep 17 00:00:00 2001 From: Benjamin Brock Date: Wed, 23 Sep 2026 11:43:40 -0700 Subject: [PATCH 15/16] Restore C++ support and integration tests --- CMakeLists.txt | 4 +- examples/CMakeLists.txt | 2 + include/binsparse/array.h | 4 + include/binsparse/detail/cpp/array.hpp | 133 +++++++++++++++++++++++++ test/bash/CMakeLists.txt | 6 ++ 5 files changed, 148 insertions(+), 1 deletion(-) create mode 100644 include/binsparse/detail/cpp/array.hpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 2e060fc..3bf70c9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,15 +3,17 @@ # SPDX-License-Identifier: BSD-3-Clause cmake_minimum_required(VERSION 3.20) -project(binsparse VERSION 1.0.0 LANGUAGES C) +project(binsparse VERSION 1.0.0) cmake_policy(SET CMP0079 NEW) set(CMAKE_C_STANDARD 11) +set(CMAKE_CXX_STANDARD 20) # FIXME: -march=native is not portable # set(CMAKE_C_FLAGS "-O3 -march=native") set(CMAKE_C_FLAGS "-g ") +set(CMAKE_CXX_FLAGS "-g ") option(ENABLE_SANITIZERS "Enable Clang sanitizers" OFF) diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 0c9ce2c..e090d7c 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -20,3 +20,5 @@ add_example(benchmark_read) add_example(benchmark_read_parallel) add_example(benchmark_write) add_example(tensor_test) + +add_subdirectory(cpp) diff --git a/include/binsparse/array.h b/include/binsparse/array.h index be0e3d3..02b063c 100644 --- a/include/binsparse/array.h +++ b/include/binsparse/array.h @@ -402,3 +402,7 @@ static inline bool bsp_array_equal(bsp_array_t x, bsp_array_t y) { } #endif + +#ifdef __cplusplus +#include +#endif diff --git a/include/binsparse/detail/cpp/array.hpp b/include/binsparse/detail/cpp/array.hpp new file mode 100644 index 0000000..9842cef --- /dev/null +++ b/include/binsparse/detail/cpp/array.hpp @@ -0,0 +1,133 @@ +// SPDX-FileCopyrightText: 2024 Binsparse Developers +// +// SPDX-License-Identifier: BSD-3-Clause + +#pragma once + +#ifdef __cplusplus + +#include +#include +#include + +namespace binsparse { + +namespace __detail { + +using array_ptr_variant_t = + std::variant; + +inline array_ptr_variant_t get_typed_ptr(bsp_array_t array) { + if (array.type == BSP_UINT8) { + uint8_t* data = (uint8_t*) array.data; + return data; + } else if (array.type == BSP_UINT16) { + uint16_t* data = (uint16_t*) array.data; + return data; + } else if (array.type == BSP_UINT32) { + uint32_t* data = (uint32_t*) array.data; + return data; + } else if (array.type == BSP_UINT64) { + uint64_t* data = (uint64_t*) array.data; + return data; + } else if (array.type == BSP_INT8) { + int8_t* data = (int8_t*) array.data; + return data; + } else if (array.type == BSP_INT16) { + int16_t* data = (int16_t*) array.data; + return data; + } else if (array.type == BSP_INT32) { + int32_t* data = (int32_t*) array.data; + return data; + } else if (array.type == BSP_INT64) { + int64_t* data = (int64_t*) array.data; + return data; + } else if (array.type == BSP_FLOAT32) { + float* data = (float*) array.data; + return data; + } else if (array.type == BSP_FLOAT64) { + double* data = (double*) array.data; + return data; + } else if (array.type == BSP_BINT8) { + int8_t* data = (int8_t*) array.data; + return data; + } else if (array.type == BSP_COMPLEX_FLOAT32) { + float _Complex* data = (float _Complex*) array.data; + return data; + } else if (array.type == BSP_COMPLEX_FLOAT64) { + double _Complex* data = (double _Complex*) array.data; + return data; + } + return {}; +} + +template +constexpr static bool is_c_complex_v = + std::is_same_v, _Complex double> || + std::is_same_v, _Complex float>; + +template +constexpr static bool mixed_c_complex_v = + (is_c_complex_v && !is_c_complex_v) || + (!is_c_complex_v && is_c_complex_v); + +} // namespace __detail + +} // namespace binsparse + +// value = array[index] +template +inline void bsp_array_read(bsp_array_t array, size_t index, T& value) { + auto variant_ptr = binsparse::__detail::get_typed_ptr(array); + + std::visit( + [&](auto* ptr) { + using U = std::remove_pointer_t; + + if constexpr (std::is_convertible_v && + !binsparse::__detail::mixed_c_complex_v) { + value = ptr[index]; + } + }, + variant_ptr); +} + +// array[index] = value +template +inline void bsp_array_write(bsp_array_t array, size_t index, U value) { + auto variant_ptr = binsparse::__detail::get_typed_ptr(array); + + std::visit( + [&](auto* ptr) { + using T = std::remove_pointer_t; + + if constexpr (std::is_convertible_v && + !binsparse::__detail::mixed_c_complex_v) { + ptr[index] = value; + } + }, + variant_ptr); +} + +// array_0[index_0] = array_1[index_1] +inline void bsp_array_awrite(bsp_array_t array_0, size_t index_0, + bsp_array_t array_1, size_t index_1) { + auto variant_ptr_0 = binsparse::__detail::get_typed_ptr(array_0); + auto variant_ptr_1 = binsparse::__detail::get_typed_ptr(array_1); + + std::visit( + [&](auto* ptr_0, auto* ptr_1) { + using T = std::remove_pointer_t; + using U = std::remove_pointer_t; + + if constexpr (std::is_convertible_v && + !binsparse::__detail::mixed_c_complex_v) { + ptr_0[index_0] = ptr_1[index_1]; + } + }, + variant_ptr_0, variant_ptr_1); +} + +#endif diff --git a/test/bash/CMakeLists.txt b/test/bash/CMakeLists.txt index 7586317..dc6ba32 100644 --- a/test/bash/CMakeLists.txt +++ b/test/bash/CMakeLists.txt @@ -35,4 +35,10 @@ if(BASH_PROGRAM) add_test(NAME integration.mouse_gene COMMAND ${BASH_PROGRAM} ${CMAKE_CURRENT_SOURCE_DIR}/test.sh ${CMAKE_BINARY_DIR}/data/mouse_gene.tar.gz) add_test(NAME integration.IMDB COMMAND ${BASH_PROGRAM} ${CMAKE_CURRENT_SOURCE_DIR}/test.sh ${CMAKE_BINARY_DIR}/data/IMDB.tar.gz) add_test(NAME integration.dwg961a COMMAND ${BASH_PROGRAM} ${CMAKE_CURRENT_SOURCE_DIR}/test.sh ${CMAKE_BINARY_DIR}/data/dwg961a.tar.gz) + + add_test(NAME integration.cpp.chesapeake COMMAND ${BASH_PROGRAM} ${CMAKE_CURRENT_SOURCE_DIR}/test-cpp.sh ${CMAKE_BINARY_DIR}/data/chesapeake.tar.gz) + add_test(NAME integration.cpp.1138_bus COMMAND ${BASH_PROGRAM} ${CMAKE_CURRENT_SOURCE_DIR}/test-cpp.sh ${CMAKE_BINARY_DIR}/data/1138_bus.tar.gz) + add_test(NAME integration.cpp.mouse_gene COMMAND ${BASH_PROGRAM} ${CMAKE_CURRENT_SOURCE_DIR}/test-cpp.sh ${CMAKE_BINARY_DIR}/data/mouse_gene.tar.gz) + add_test(NAME integration.cpp.IMDB COMMAND ${BASH_PROGRAM} ${CMAKE_CURRENT_SOURCE_DIR}/test-cpp.sh ${CMAKE_BINARY_DIR}/data/IMDB.tar.gz) + add_test(NAME integration.cpp.dwg961a COMMAND ${BASH_PROGRAM} ${CMAKE_CURRENT_SOURCE_DIR}/test-cpp.sh ${CMAKE_BINARY_DIR}/data/dwg961a.tar.gz) endif() From 6266bc8e5102e1740ba37e80026a9f80de0b0c6f Mon Sep 17 00:00:00 2001 From: Benjamin Brock Date: Wed, 23 Sep 2026 12:45:15 -0700 Subject: [PATCH 16/16] Restore Julia tensor testing and CI setup --- .github/workflows/ci.yml | 10 +++++ test/CMakeLists.txt | 1 + test/julia/CMakeLists.txt | 13 ++++++ test/julia/tensor_test.jl | 95 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 119 insertions(+) create mode 100644 test/julia/CMakeLists.txt create mode 100644 test/julia/tensor_test.jl diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 98cf896..4c28a88 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,6 +41,8 @@ jobs: CXX: ${{ matrix.cxx }} steps: - uses: actions/checkout@v4 + - name: Install Julia + run: curl -fsSL https://install.julialang.org | sh -s -- -y - name: CMake run: | sudo apt-get update @@ -85,6 +87,14 @@ jobs: cache: false - name: Matrix Tests run: ctest --test-dir ./build/test/bash + - name: Install Julia + run: curl -fsSL https://install.julialang.org | sh -s -- -y + - name: Disable Julia precompilation + run: julia -e 'using Pkg; Pkg.add(["PrecompileTools", "Preferences"]); using PrecompileTools, Preferences; set_preferences!(PrecompileTools, "precompile_workloads" => false; force=true)' + - name: Install Julia Packages + run: julia -e 'using Pkg; Pkg.add(["Finch", "HDF5"])' + - name: Tensor Tests + run: ctest --test-dir ./build/test/julia - name: Compliance Tests env: BINSPARSE_BUILD_DIR: ${{ github.workspace }}/build diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index abc3905..bc81d56 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -4,3 +4,4 @@ add_subdirectory(bash) add_subdirectory(c) +add_subdirectory(julia) diff --git a/test/julia/CMakeLists.txt b/test/julia/CMakeLists.txt new file mode 100644 index 0000000..4c2c9d8 --- /dev/null +++ b/test/julia/CMakeLists.txt @@ -0,0 +1,13 @@ +# SPDX-FileCopyrightText: 2024 Binsparse Developers +# +# SPDX-License-Identifier: BSD-3-Clause + +find_program(JULIA_PROGRAM julia) + +enable_testing() + +if(JULIA_PROGRAM) + add_test(NAME tensors.tensor_test COMMAND ${JULIA_PROGRAM} ${CMAKE_CURRENT_SOURCE_DIR}/tensor_test.jl) + + add_test(NAME tensors.cpp.tensor_test COMMAND ${JULIA_PROGRAM} ${CMAKE_CURRENT_SOURCE_DIR}/tensor_test.jl cpp) +endif() diff --git a/test/julia/tensor_test.jl b/test/julia/tensor_test.jl new file mode 100644 index 0000000..cbba45f --- /dev/null +++ b/test/julia/tensor_test.jl @@ -0,0 +1,95 @@ +# SPDX-FileCopyrightText: 2024 Binsparse Developers +# +# SPDX-License-Identifier: BSD-3-Clause + +using Finch; +using HDF5; + +dir = pwd(); + +if length(ARGS) == 0 + tensor_test = joinpath(dir, "../../examples/tensor_test") + test_files = joinpath(dir, "../../tensor_test_files") +else + tensor_test = joinpath(dir, "../../examples/cpp/tensor_test-cpp") + test_files = joinpath(dir, "../../tensor_test_files/cpp") +end + +mkpath(test_files) + +function tensortest(tensor::Tensor, input::AbstractString, output::AbstractString) + print("tensor_test ", input, " -> ", output, "\n") + fwrite(input, tensor) + run(`$tensor_test $input $output`) + output_tensor = fread(output) + @assert tensor == output_tensor +end + +tensortest( + Tensor( + Dense(SparseList{Int32}(SparseList{Int32}(Element{0.0,Float64,Int32}()))), + fsprand(10, 10, 10, 0.1) + ), + joinpath(test_files, "input1.bsp.h5"), + joinpath(test_files, "output1.bsp.h5") +) + + +tensortest( + Tensor( + Dense(SparseCOO{2}(Element(0.0))), + fsprand(10, 10, 10, 0.1) + ), + joinpath(test_files, "input2.bsp.h5"), + joinpath(test_files, "output2.bsp.h5") +) + +tensortest( + Tensor( + Dense(Dense(Dense(Element{0.0,Float64,Int32}()))), + fsprand(10, 10, 10, 0.1) + ), + joinpath(test_files, "input3.bsp.h5"), + joinpath(test_files, "output3.bsp.h5") +) + +tensortest( + Tensor( + SparseCOO{2}(Element(0.0)), + fsprand(10, 10, 0.1) + ), + joinpath(test_files, "input4.bsp.h5"), + joinpath(test_files, "output4.bsp.h5") +) + +tensortest( + Tensor( + Dense(SparseList{Int32}(Element{0.0,Float64,Int32}())), + [ + 0 1 0 3; + 1 0 0 4; + ] + ), + joinpath(test_files, "inputcsr.bsp.h5"), + joinpath(test_files, "outputcsr.bsp.h5") +) +tensortest( + Tensor( + Dense(SparseList{Int32}(Element{0.0,Float64,Int32}())), + fsprand(10, 10, 0.1) + ), + joinpath(test_files, "inputcsr2.bsp.h5"), + joinpath(test_files, "outputcsr2.bsp.h5") +) + +tensortest( + Tensor( + Dense(Dense(Element(0.0))), + [ + 1 1 2 3; + 6 1 5 4; + ] + ), + joinpath(test_files, "inputdense.bsp.h5"), + joinpath(test_files, "outputdense.bsp.h5") +)