Skip to content

[common] Fix CHAR min/max statistics collection #4080

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
beetle0915 wants to merge 4 commits into
base: main
Choose a base branch
from
Open

[common] Fix CHAR min/max statistics collection #4080

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
c606394
[common] Collect min/max statistics for CHAR
beetle0915 Aug 24, 2026
0f6bd6a
[common] Serialize CHAR statistics
beetle0915 Aug 24, 2026
2dd128d
[common][test] Test CHAR statistics round trip
beetle0915 Aug 24, 2026
54019b9
[server][test] Test CHAR statistics pruning
beetle0915 Aug 24, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions fluss-common/src/main/java/org/apache/fluss/record/LogRecordBatchStatisticsCollector.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,8 @@
import java.util.Arrays;
import java.util.function.Function;

import static org.apache.fluss.types.DataTypeChecks.getLength;

/**
* Collector for {@link LogRecordBatchStatistics} that accumulates statistics during record batch
* construction. Manages statistics data in memory arrays and supports schema-aware statistics
Expand Down Expand Up @@ -160,6 +162,11 @@ private void updateMinMax(int statsIndex, int schemaIndex, InternalRow row) {
double doubleValue = row.getDouble(schemaIndex);
updateMinMaxInternal(statsIndex, doubleValue, Double::compare);
break;
case CHAR:
BinaryString charValue = row.getChar(schemaIndex, getLength(fieldType));
updateMinMaxInternal(
statsIndex, charValue, BinaryString::compareTo, BinaryString::copy);
break;
case STRING:
BinaryString stringValue = row.getString(schemaIndex);
updateMinMaxInternal(
Expand Down
5 changes: 5 additions & 0 deletions fluss-common/src/main/java/org/apache/fluss/record/LogRecordBatchStatisticsWriter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
import java.io.IOException;

import static org.apache.fluss.record.LogRecordBatchFormat.STATISTICS_VERSION;
import static org.apache.fluss.types.DataTypeChecks.getLength;

/**
* A high-performance writer for LogRecordBatch statistics that efficiently serializes statistical
Expand Down Expand Up @@ -205,6 +206,10 @@ private AlignedRow convertToAlignedRow(InternalRow row) {
case DOUBLE:
reusableRowWriter.writeDouble(i, row.getDouble(i));
break;
case CHAR:
int charLength = getLength(fieldType);
reusableRowWriter.writeChar(i, row.getChar(i, charLength), charLength);
break;
case STRING:
reusableRowWriter.writeString(i, row.getString(i));
break;
Expand Down
28 changes: 28 additions & 0 deletions ...s-common/src/test/java/org/apache/fluss/record/LogRecordBatchStatisticsCollectorTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -169,6 +169,34 @@ void testProcessNullValues() throws IOException {
assertThat(statistics.getMaxValues().getInt(0)).isEqualTo(3);
}

@Test
void testCharStatisticsSerializationRoundTrip() throws IOException {
RowType charRowType = DataTypes.ROW(new DataField("char_val", DataTypes.CHAR(3)));
LogRecordBatchStatisticsCollector charCollector =
new LogRecordBatchStatisticsCollector(charRowType, new int[] {0});

List<Object[]> charData =
Arrays.asList(
new Object[] {"cat"},
new Object[] {"a"},
new Object[] {null},
new Object[] {"dog"});
for (Object[] data : charData) {
charCollector.processRow(DataTestUtils.row(data));
}

MemorySegment segment = MemorySegment.allocateHeapMemory(1024);
charCollector.writeStatistics(new MemorySegmentOutputView(segment));

DefaultLogRecordBatchStatistics statistics =
LogRecordBatchStatisticsParser.parseStatistics(segment, 0, charRowType, 1);

assertThat(statistics.getMinValues().getChar(0, 3)).isEqualTo(BinaryString.fromString("a"));
assertThat(statistics.getMaxValues().getChar(0, 3))
.isEqualTo(BinaryString.fromString("dog"));
assertThat(statistics.getNullCounts()[0]).isEqualTo(1);
}

@Test
void testPartialStatsIndexMapping() throws IOException {
RowType fullRowType =
Expand Down
21 changes: 21 additions & 0 deletions fluss-server/src/test/java/org/apache/fluss/server/log/RecordBatchFilterTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,27 @@ void testSchemaAwareStatisticsWithNullStats() {
assertThat(result).isTrue();
}

@Test
void testCharStatisticsCanPruneRecordBatch() throws IOException {
RowType rowType = RowType.of(DataTypes.CHAR(3));
LogRecordBatchStatisticsCollector collector =
new LogRecordBatchStatisticsCollector(rowType, new int[] {0});
collector.processRow(GenericRow.of(BinaryString.fromString("cat")));
collector.processRow(GenericRow.of(BinaryString.fromString("ant")));
collector.processRow(GenericRow.of(BinaryString.fromString("dog")));

MemorySegmentOutputView outputView = new MemorySegmentOutputView(1024);
collector.writeStatistics(outputView);
DefaultLogRecordBatchStatistics statistics =
LogRecordBatchStatisticsParser.parseStatistics(
outputView.getMemorySegment(), 0, rowType, TEST_SCHEMA_ID);

Predicate predicate =
new PredicateBuilder(rowType).greaterThan(0, BinaryString.fromString("zoo"));

assertThat(mayMatch(predicate, TEST_SCHEMA_ID, null, 3L, statistics)).isFalse();
}

// =====================================================
// Schema Evolution Tests
// =====================================================
Expand Down
Loading