Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions compat/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@ summary.
| `javaWrite`/`cppWrite` | Whether that language writes the fixture. |
| `javaRead`/`cppRead` | Whether that language validates the fixture when the other side wrote it. |
| `gapReason` | Required whenever any write/read cell is `no`; the reason is reported in every CI run. |
| `timestampProfile` | `increasing` (default; the timestamp equals the row index) or `duplicates` (the timestamp is `row/2`, so every value appears twice, written with `allow_non_decreasing_timestamps` enabled). This trailing column is optional; omitting it means `increasing`. |

A case with a `no` in a write column is an explicit declared gap. A case absent
from the file is not a gap: the completeness test in both suites enumerates the
Expand Down
4 changes: 3 additions & 1 deletion compat/compatibility-matrix.csv
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@
# 'sprintz' profile (values within +/-1e9). Streaming numeric extremes are
# covered under PLAIN, RLE, TS_2DIFF, GORILLA, CHIMP and RLBE.

id,model,dataType,encoding,compressions,rowCount,pageRows,chunks,devices,valueProfile,nullPatterns,javaWrite,cppWrite,javaRead,cppRead,gapReason
id,model,dataType,encoding,compressions,rowCount,pageRows,chunks,devices,valueProfile,nullPatterns,javaWrite,cppWrite,javaRead,cppRead,gapReason,timestampProfile
table_int32_plain_r10,table,INT32,PLAIN,ALL,10,4,1,1,default,NONE,yes,yes,yes,yes,
table_int32_plain_r1000,table,INT32,PLAIN,ALL,1000,100,1,1,default,NONE,yes,yes,yes,yes,
table_int32_plain_r100000,table,INT32,PLAIN,ALL,100000,10000,1,1,default,NONE,yes,yes,yes,yes,
Expand Down Expand Up @@ -342,3 +342,5 @@ table_two_pages,table,FLOAT,CHIMP,ALL,250,100,1,1,default,NONE,yes,yes,yes,yes,
table_multi_chunk,table,INT64,TS_2DIFF,ALL,300,0,3,1,default,NONE,yes,yes,yes,yes,
table_multi_chunk_group,table,INT32,RLE,ALL,50,0,1,3,runs,NONE,yes,yes,yes,yes,
table_long_runs,table,INT64,RLE,ALL,1024,0,1,1,runs,NONE,yes,yes,yes,yes,
table_int32_plain_dups,table,INT32,PLAIN,UNCOMPRESSED,10,4,1,1,default,NONE,yes,yes,yes,yes,,duplicates
tree_int32_plain_dups,tree,INT32,PLAIN,UNCOMPRESSED,10,4,1,1,default,NONE,yes,yes,yes,yes,,duplicates
7 changes: 7 additions & 0 deletions cpp/src/common/config/config.h
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,13 @@ typedef struct ConfigValue {
// throughput: writes return success as soon as data is in the page cache.
// Only set to false if the caller drives its own fsync policy.
bool sync_on_close_ = true;
// When true, a timestamp equal to the previous one is accepted; only a
// strictly decreasing timestamp is rejected with E_OUT_OF_ORDER. Equal
// timestamps are preserved as separate rows (no write-time dedup). Like
// thread_count_/sync_on_close_ it keeps its in-class default rather than
// being reset by init_config_value(), so a setter called before
// libtsfile_init() is not clobbered.
bool allow_non_decreasing_timestamps_ = false;
} ConfigValue;

extern void init_config_value();
Expand Down
8 changes: 8 additions & 0 deletions cpp/src/common/constant/tsfile_constant.h
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,14 @@ static const std::string TIME_COLUMN_NAME = "time";
static const unsigned char VALUE_COLUMN_MASK = 0x40;
static const int NO_STR_TO_READ = -1;

// File-level property written when the writer is created with
// allow_non_decreasing_timestamps_ enabled. Its presence (value "true") tells
// downstream readers that the file may contain duplicate timestamps, so they
// must not assume strictly increasing times within a series.
static const std::string TSFILE_PROP_ALLOW_NON_DECREASING_TIMESTAMPS =
"allowNonDecreasingTimestamps";
static const std::string TSFILE_PROP_VALUE_TRUE = "true";

// Identifier characters: ASCII alphanumerics / underscore plus any non-ASCII
// (high) byte. The \x80-\xFF byte range covers UTF-8 lead/continuation bytes
// (e.g. CJK characters) and is accepted by every std::regex implementation;
Expand Down
8 changes: 8 additions & 0 deletions cpp/src/common/global.h
Original file line number Diff line number Diff line change
Expand Up @@ -209,6 +209,14 @@ FORCE_INLINE bool get_parallel_write_enabled() {
return g_config_value_.parallel_write_enabled_;
}

FORCE_INLINE void set_allow_non_decreasing_timestamps(bool enabled) {
g_config_value_.allow_non_decreasing_timestamps_ = enabled;
}

FORCE_INLINE bool get_allow_non_decreasing_timestamps() {
return g_config_value_.allow_non_decreasing_timestamps_;
}

// Select the backend used by subsequently opened local files. Existing
// LocalRandomAccessReadFile instances retain the backend selected when opened.
// This setting deliberately lives outside exported ConfigValue so adding it
Expand Down
5 changes: 5 additions & 0 deletions cpp/src/cwrapper/tsfile_cwrapper.cc
Original file line number Diff line number Diff line change
Expand Up @@ -146,6 +146,11 @@ int set_global_compression(uint8_t compression) {
return common::set_global_compression(compression);
}

int set_allow_non_decreasing_timestamps(bool enabled) {
common::set_allow_non_decreasing_timestamps(enabled);
return common::E_OK;
}

ERRNO tsfile_set_file_read_backend(int32_t backend) {
switch (backend) {
case TSFILE_READ_BACKEND_AUTO:
Expand Down
14 changes: 14 additions & 0 deletions cpp/src/cwrapper/tsfile_cwrapper.h
Original file line number Diff line number Diff line change
Expand Up @@ -433,6 +433,20 @@ int set_datatype_encoding(uint8_t data_type, uint8_t encoding);
*/
int set_global_compression(uint8_t compression);

/**
* @brief Allow equal (non-decreasing) timestamps when writing.
*
* When enabled, a timestamp equal to the previously written timestamp is
* accepted; only a strictly decreasing timestamp is rejected with
* E_OUT_OF_ORDER. Duplicate timestamps are preserved as separate rows (no
* write-time deduplication). Default is false, which rejects both equal and
* decreasing timestamps.
*
* @param enabled true to allow equal timestamps, false to keep strict ordering
* @return E_OK
*/
int set_allow_non_decreasing_timestamps(bool enabled);

/*--------------------------TsFile Reader and Writer------------------------ */

/**
Expand Down
11 changes: 11 additions & 0 deletions cpp/src/file/tsfile_io_writer.cc
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
#include <limits>
#include <memory>

#include "common/constant/tsfile_constant.h"
#include "common/device_id.h"
#include "common/global.h"
#include "common/logger/elog.h"
Expand Down Expand Up @@ -53,6 +54,16 @@ int TsFileIOWriter::init(WriteFile* write_file) {
meta_allocator_.init(page_size, MOD_TSFILE_WRITER_META);
chunk_meta_count_ = 0;
file_ = write_file;
// Record the writer's timestamp policy so downstream readers (e.g. an
// IoTDB load) can tell that series in this file may contain duplicate
// timestamps. Only written when the flag is on, so flag-off files keep
// their metadata byte-for-byte unchanged.
if (common::g_config_value_.allow_non_decreasing_timestamps_) {
tsfile_properties_[TSFILE_PROP_ALLOW_NON_DECREASING_TIMESTAMPS] =
TsFilePropertyValue(
reinterpret_cast<const uint8_t*>(TSFILE_PROP_VALUE_TRUE.data()),
static_cast<uint32_t>(TSFILE_PROP_VALUE_TRUE.size()));
}
// Re-arm destroy() for the new lifecycle. Without this, a writer that
// was destroy()'d and then init()'d again would leak the fresh
// meta_allocator_/write_stream_/file_ on its next destroy().
Expand Down
15 changes: 12 additions & 3 deletions cpp/src/writer/time_page_writer.h
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
#define WRITER_PAGE_TIME_WRITER_H

#include "common/allocator/byte_stream.h"
#include "common/global.h"
#include "common/statistic.h"
#include "compress/compressor.h"
#include "encoding/encoder.h"
Expand Down Expand Up @@ -74,7 +75,7 @@ class TimePageWriter {
FORCE_INLINE int write(int64_t timestamp) {
int ret = common::E_OK;
if (statistic_->count_ != 0 && is_inited_ &&
timestamp <= statistic_->end_time_) {
is_out_of_order(timestamp, statistic_->end_time_)) {
return common::E_OUT_OF_ORDER;
}
if (RET_FAIL(time_encoder_->encode(timestamp, time_out_stream_))) {
Expand All @@ -89,12 +90,12 @@ class TimePageWriter {
if (count == 0) return ret;
// Check order: first timestamp vs existing end_time
if (statistic_->count_ != 0 && is_inited_ &&
timestamps[0] <= statistic_->end_time_) {
is_out_of_order(timestamps[0], statistic_->end_time_)) {
return common::E_OUT_OF_ORDER;
}
// Check monotonicity within batch
for (uint32_t i = 1; i < count; i++) {
if (timestamps[i] <= timestamps[i - 1]) {
if (is_out_of_order(timestamps[i], timestamps[i - 1])) {
return common::E_OUT_OF_ORDER;
}
}
Expand Down Expand Up @@ -135,6 +136,14 @@ class TimePageWriter {
void destroy_page_data() { cur_page_data_.destroy(); }

private:
// Reject a timestamp only when it is strictly less than the previous one
// (allow_non_decreasing_timestamps_ enabled), or at most equal to it
// (flag disabled, the historical behavior).
FORCE_INLINE static bool is_out_of_order(int64_t time, int64_t last_time) {
return common::g_config_value_.allow_non_decreasing_timestamps_
? time < last_time
: time <= last_time;
}
FORCE_INLINE int prepare_end_page() {
int ret = common::E_OK;
if (RET_FAIL(time_encoder_->flush(time_out_stream_))) {
Expand Down
74 changes: 32 additions & 42 deletions cpp/src/writer/tsfile_writer.cc
Original file line number Diff line number Diff line change
Expand Up @@ -785,18 +785,28 @@ int TsFileWriter::check_memory_size_and_may_flush_chunks() {
return ret;
}

bool TsFileWriter::recovered_last_time_order_violated(
const std::shared_ptr<IDeviceID>& device_id, int64_t timestamp) const {
if (!enforce_recovered_last_time_order_) {
return false;
}
auto schema_it = schemas_.find(device_id);
if (schema_it == schemas_.end() || schema_it->second == nullptr) {
return false;
}
return common::g_config_value_.allow_non_decreasing_timestamps_
? timestamp < schema_it->second->last_time_
: timestamp <= schema_it->second->last_time_;
}

int TsFileWriter::write_record(const TsRecord& record) {
if (UNLIKELY(unrecoverable_)) return E_DATA_INCONSISTENCY;
int ret = E_OK;
auto device_id = std::make_shared<StringArrayDeviceID>(record.device_id_);
// After recovery, refuse writes whose timestamp would land at or before
// any already-flushed chunk's end_time for this device.
if (enforce_recovered_last_time_order_) {
auto schema_it = schemas_.find(device_id);
if (schema_it != schemas_.end() && schema_it->second != nullptr &&
record.timestamp_ <= schema_it->second->last_time_) {
return E_OUT_OF_ORDER;
}
if (recovered_last_time_order_violated(device_id, record.timestamp_)) {
return E_OUT_OF_ORDER;
}
// std::vector<ChunkWriter*> chunk_writers;
SimpleVector<ChunkWriter*> chunk_writers;
Expand Down Expand Up @@ -834,12 +844,8 @@ int TsFileWriter::write_record_aligned(const TsRecord& record) {
if (UNLIKELY(unrecoverable_)) return E_DATA_INCONSISTENCY;
int ret = E_OK;
auto device_id = std::make_shared<StringArrayDeviceID>(record.device_id_);
if (enforce_recovered_last_time_order_) {
auto schema_it = schemas_.find(device_id);
if (schema_it != schemas_.end() && schema_it->second != nullptr &&
record.timestamp_ <= schema_it->second->last_time_) {
return E_OUT_OF_ORDER;
}
if (recovered_last_time_order_violated(device_id, record.timestamp_)) {
return E_OUT_OF_ORDER;
}
SimpleVector<ValueChunkWriter*> value_chunk_writers;
SimpleVector<common::TSDataType> data_types;
Expand Down Expand Up @@ -1056,13 +1062,9 @@ int TsFileWriter::write_tablet_aligned(const Tablet& tablet) {
auto device_id =
std::make_shared<StringArrayDeviceID>(tablet.insert_target_name_);
const uint32_t total_rows = tablet.get_cur_row_size();
if (enforce_recovered_last_time_order_ && total_rows > 0 &&
tablet.timestamps_ != nullptr) {
auto schema_it = schemas_.find(device_id);
if (schema_it != schemas_.end() && schema_it->second != nullptr &&
tablet.timestamps_[0] <= schema_it->second->last_time_) {
return E_OUT_OF_ORDER;
}
if (total_rows > 0 && tablet.timestamps_ != nullptr &&
recovered_last_time_order_violated(device_id, tablet.timestamps_[0])) {
return E_OUT_OF_ORDER;
}
SimpleVector<ValueChunkWriter*> value_chunk_writers;
TimeChunkWriter* time_chunk_writer = nullptr;
Expand Down Expand Up @@ -1206,13 +1208,9 @@ int TsFileWriter::write_tablet(const Tablet& tablet) {
// and would let uninitialized timestamps/values past the live range leak
// into the chunk.
const uint32_t total_rows = tablet.get_cur_row_size();
if (enforce_recovered_last_time_order_ && total_rows > 0 &&
tablet.timestamps_ != nullptr) {
auto schema_it = schemas_.find(device_id);
if (schema_it != schemas_.end() && schema_it->second != nullptr &&
tablet.timestamps_[0] <= schema_it->second->last_time_) {
return E_OUT_OF_ORDER;
}
if (total_rows > 0 && tablet.timestamps_ != nullptr &&
recovered_last_time_order_violated(device_id, tablet.timestamps_[0])) {
return E_OUT_OF_ORDER;
}
SimpleVector<ChunkWriter*> chunk_writers;
SimpleVector<common::TSDataType> data_types;
Expand Down Expand Up @@ -1336,14 +1334,10 @@ int TsFileWriter::write_table(Tablet& tablet) {
// Recovery: refuse any segment whose first timestamp would land
// at or before a flushed chunk's end_time for this device. This
// mirrors the per-record / per-tablet check on the tree path.
if (enforce_recovered_last_time_order_ && tablet.timestamps_ &&
ei > si) {
auto schema_it = schemas_.find(device_id);
if (schema_it != schemas_.end() &&
schema_it->second != nullptr &&
tablet.timestamps_[si] <= schema_it->second->last_time_) {
return E_OUT_OF_ORDER;
}
if (tablet.timestamps_ && ei > si &&
recovered_last_time_order_violated(device_id,
tablet.timestamps_[si])) {
return E_OUT_OF_ORDER;
}
auto idx_it = device_ctx_index.find(device_id);
if (idx_it == device_ctx_index.end()) {
Expand Down Expand Up @@ -1565,14 +1559,10 @@ int TsFileWriter::write_table(Tablet& tablet) {
if (end_idx == 0) continue;

const uint32_t si = static_cast<uint32_t>(start_idx);
if (enforce_recovered_last_time_order_ && tablet.timestamps_ &&
end_idx > start_idx) {
auto schema_it = schemas_.find(device_id);
if (schema_it != schemas_.end() &&
schema_it->second != nullptr &&
tablet.timestamps_[si] <= schema_it->second->last_time_) {
return E_OUT_OF_ORDER;
}
if (tablet.timestamps_ && end_idx > start_idx &&
recovered_last_time_order_violated(device_id,
tablet.timestamps_[si])) {
return E_OUT_OF_ORDER;
}
MeasurementNamesFromTablet mnames_getter(tablet);
SimpleVector<ChunkWriter*> chunk_writers;
Expand Down
8 changes: 8 additions & 0 deletions cpp/src/writer/tsfile_writer.h
Original file line number Diff line number Diff line change
Expand Up @@ -227,6 +227,14 @@ class TsFileWriter {
// flush/close refuse — can be unit-tested directly.
friend class TsFileWriterUnrecoverableTest;

// After recovery (enforce_recovered_last_time_order_), returns true when
// `timestamp` must be rejected because it would land at or before a
// flushed chunk's end_time for this device. With
// allow_non_decreasing_timestamps_ enabled only strictly decreasing
// timestamps are rejected; otherwise equality is rejected too.
bool recovered_last_time_order_violated(
const std::shared_ptr<IDeviceID>& device_id, int64_t timestamp) const;

int write_column_batch(storage::ChunkWriter* chunk_writer,
const Tablet& tablet, int col_idx,
uint32_t start_idx, uint32_t end_idx);
Expand Down
Loading
Loading