diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/consensus/local/tablemodel/ConsensusSubscriptionTableITSupport.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/consensus/local/tablemodel/ConsensusSubscriptionTableITSupport.java index d319710e3830a..52b15f258e750 100644 --- a/integration-test/src/test/java/org/apache/iotdb/subscription/it/consensus/local/tablemodel/ConsensusSubscriptionTableITSupport.java +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/consensus/local/tablemodel/ConsensusSubscriptionTableITSupport.java @@ -32,6 +32,8 @@ import org.apache.tsfile.read.common.RowRecord; import org.apache.tsfile.read.query.dataset.ResultSet; +import org.awaitility.Awaitility; +import org.awaitility.core.ConditionFactory; import org.junit.Assert; import java.time.Duration; @@ -44,6 +46,7 @@ import java.util.Map; import java.util.Properties; import java.util.Set; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; final class ConsensusSubscriptionTableITSupport { @@ -52,6 +55,7 @@ final class ConsensusSubscriptionTableITSupport { private static final AtomicInteger IDENTIFIER = new AtomicInteger(0); private static final Duration DEFAULT_POLL_TIMEOUT = Duration.ofSeconds(1); + private static final Duration DEFAULT_DRAIN_TIMEOUT = Duration.ofMinutes(2); private static final int QUIET_ROUNDS_AFTER_DATA = 3; private static final int QUIET_ROUNDS_WITHOUT_DATA = 8; @@ -116,7 +120,7 @@ static void createConsensusTopic( final String topicName, final String databasePattern, final String tablePattern, - final String columnPattern) + final String columnFilter) throws Exception { final String host = EnvFactory.getEnv().getIP(); final int port = Integer.parseInt(EnvFactory.getEnv().getPort()); @@ -131,13 +135,28 @@ static void createConsensusTopic( config.put(TopicConstant.FORMAT_KEY, TopicConstant.FORMAT_SESSION_DATA_SETS_HANDLER_VALUE); config.put(TopicConstant.DATABASE_KEY, databasePattern); config.put(TopicConstant.TABLE_KEY, tablePattern); - if (columnPattern != null) { - config.put(TopicConstant.COLUMN_KEY, columnPattern); + if (columnFilter != null) { + config.put(TopicConstant.COLUMN_FILTER_KEY, columnFilter); } session.createTopic(topicName, config); } } + static void alterConsensusTopicColumnFilter(final String topicName, final String columnFilter) + throws Exception { + final String host = EnvFactory.getEnv().getIP(); + final int port = Integer.parseInt(EnvFactory.getEnv().getPort()); + + try (final ISubscriptionTableSession session = + new SubscriptionTableSessionBuilder().host(host).port(port).build()) { + session.open(); + + final Properties config = new Properties(); + config.put(TopicConstant.COLUMN_FILTER_KEY, columnFilter); + session.alterTopic(topicName, config); + } + } + static SubscriptionTablePullConsumer createConsumer( final String consumerId, final String consumerGroupId) throws Exception { final SubscriptionTablePullConsumer consumer = @@ -191,6 +210,49 @@ static Set insertRows( return rowKeys; } + static Set insertRows( + final String database, + final String tableName, + final long startTimestampInclusive, + final int rowCount, + final boolean includeS2, + final boolean includeS3, + final boolean flush) + throws Exception { + final Set rowKeys = new LinkedHashSet<>(); + + try (final ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + session.executeNonQueryStatement("use " + database); + for (int row = 0; row < rowCount; row++) { + final long timestamp = startTimestampInclusive + row; + final StringBuilder columns = new StringBuilder("tag1, s1"); + final StringBuilder values = + new StringBuilder( + String.format( + Locale.ROOT, "'%s', %d", tableName + "_tag_" + timestamp, timestamp * 10L)); + if (includeS2) { + columns.append(", s2"); + values.append(String.format(Locale.ROOT, ", %.1f", timestamp + 0.5d)); + } + if (includeS3) { + columns.append(", s3"); + values.append(timestamp % 2 == 0 ? ", true" : ", false"); + } + columns.append(", time"); + values.append(", ").append(timestamp); + session.executeNonQueryStatement( + String.format( + Locale.ROOT, "insert into %s(%s) values (%s)", tableName, columns, values)); + rowKeys.add(rowKey(database, tableName, timestamp)); + } + if (flush) { + session.executeNonQueryStatement("flush"); + } + } + + return rowKeys; + } + static ConsumedRecords pollAndCommitUntilAtLeast( final SubscriptionTablePullConsumer consumer, final int expectedUniqueRows, @@ -207,28 +269,34 @@ static ConsumedRecords pollAndCommitUntilAtLeast( final Duration pollTimeout) throws Exception { final ConsumedRecords consumed = new ConsumedRecords(); - int emptyRounds = 0; - - for (int round = 0; round < maxPollRounds; round++) { - final List messages = consumer.poll(pollTimeout); - if (messages.isEmpty()) { - emptyRounds++; - if (consumed.getUniqueRowCount() >= expectedUniqueRows - && emptyRounds >= QUIET_ROUNDS_AFTER_DATA) { - break; - } - if (consumed.getUniqueRowCount() == 0 - && expectedUniqueRows == 0 - && emptyRounds >= QUIET_ROUNDS_WITHOUT_DATA) { - break; - } - continue; - } + final AtomicInteger emptyRounds = new AtomicInteger(0); + awaitDrain(maxPollRounds, pollTimeout) + .untilAsserted( + () -> { + pollAndCommitOnce(consumer, pollTimeout, consumed, emptyRounds); + Assert.assertTrue( + atLeastTimeoutMessage(expectedUniqueRows, consumed), + hasDrainedAtLeast(consumed, expectedUniqueRows, emptyRounds.get())); + }); - emptyRounds = 0; - consumed.merge(consumeMessages(messages)); - consumer.commitSync(messages); - } + return consumed; + } + + static ConsumedRecords pollAndCommitUntilContains( + final SubscriptionTablePullConsumer consumer, + final Set expectedRowKeys, + final int maxPollRounds) + throws Exception { + final ConsumedRecords consumed = new ConsumedRecords(); + final AtomicInteger emptyRounds = new AtomicInteger(0); + awaitDrain(maxPollRounds, DEFAULT_POLL_TIMEOUT) + .untilAsserted( + () -> { + pollAndCommitOnce(consumer, DEFAULT_POLL_TIMEOUT, consumed, emptyRounds); + Assert.assertTrue( + containsTimeoutMessage(expectedRowKeys, consumed), + hasDrainedExpectedKeys(consumed, expectedRowKeys, emptyRounds.get())); + }); return consumed; } @@ -251,24 +319,15 @@ static ConsumedRecords pollWithInfoAndCommitUntilAtLeast( final Duration pollTimeout) throws Exception { final ConsumedRecords consumed = new ConsumedRecords(); - int emptyRounds = 0; - - for (int round = 0; round < maxPollRounds; round++) { - final PollResult pollResult = consumer.pollWithInfo(topicNames, pollTimeout.toMillis()); - final List messages = pollResult.getMessages(); - if (messages.isEmpty()) { - emptyRounds++; - if (consumed.getUniqueRowCount() >= expectedUniqueRows - && emptyRounds >= QUIET_ROUNDS_AFTER_DATA) { - break; - } - continue; - } - - emptyRounds = 0; - consumed.merge(consumeMessages(messages)); - consumer.commitSync(messages); - } + final AtomicInteger emptyRounds = new AtomicInteger(0); + awaitDrain(maxPollRounds, pollTimeout) + .untilAsserted( + () -> { + pollWithInfoAndCommitOnce(consumer, topicNames, pollTimeout, consumed, emptyRounds); + Assert.assertTrue( + atLeastTimeoutMessage(expectedUniqueRows, consumed), + hasDrainedAtLeast(consumed, expectedUniqueRows, emptyRounds.get())); + }); return consumed; } @@ -278,7 +337,8 @@ static void assertExactRowKeys( Assert.assertTrue( "Unexpected duplicate row keys: " + consumed.getDuplicateRowKeys(), consumed.getDuplicateRowKeys().isEmpty()); - Assert.assertEquals(expectedRowKeys, consumed.getRowKeys()); + Assert.assertEquals( + rowKeyDiffMessage(expectedRowKeys, consumed), expectedRowKeys, consumed.getRowKeys()); Assert.assertEquals(expectedRowKeys.size(), consumed.getRowCount()); } @@ -384,6 +444,110 @@ private static ConsumedRecords consumeMessages(final List m return consumed; } + private static void pollAndCommitOnce( + final SubscriptionTablePullConsumer consumer, + final Duration pollTimeout, + final ConsumedRecords consumed, + final AtomicInteger emptyRounds) + throws Exception { + final List messages = consumer.poll(pollTimeout); + if (messages.isEmpty()) { + emptyRounds.incrementAndGet(); + return; + } + + emptyRounds.set(0); + consumed.merge(consumeMessages(messages)); + consumer.commitSync(messages); + } + + private static void pollWithInfoAndCommitOnce( + final SubscriptionTablePullConsumer consumer, + final Set topicNames, + final Duration pollTimeout, + final ConsumedRecords consumed, + final AtomicInteger emptyRounds) + throws Exception { + final PollResult pollResult = consumer.pollWithInfo(topicNames, pollTimeout.toMillis()); + final List messages = pollResult.getMessages(); + if (messages.isEmpty()) { + emptyRounds.incrementAndGet(); + return; + } + + emptyRounds.set(0); + consumed.merge(consumeMessages(messages)); + consumer.commitSync(messages); + } + + private static ConditionFactory awaitDrain( + final int legacyMaxPollRounds, final Duration pollTimeout) { + final Duration drainTimeout = drainTimeout(legacyMaxPollRounds, pollTimeout); + return Awaitility.await() + .pollInSameThread() + .pollDelay(0, TimeUnit.MILLISECONDS) + .pollInterval(1, TimeUnit.MILLISECONDS) + .atMost(drainTimeout.toMillis(), TimeUnit.MILLISECONDS); + } + + private static Duration drainTimeout(final int legacyMaxPollRounds, final Duration pollTimeout) { + final long legacyTimeoutMillis = + Math.max(0L, legacyMaxPollRounds) * Math.max(1L, pollTimeout.toMillis()); + final Duration legacyTimeout = Duration.ofMillis(legacyTimeoutMillis); + return legacyTimeout.compareTo(DEFAULT_DRAIN_TIMEOUT) > 0 + ? legacyTimeout + : DEFAULT_DRAIN_TIMEOUT; + } + + private static boolean hasDrainedAtLeast( + final ConsumedRecords consumed, final int expectedUniqueRows, final int emptyRounds) { + if (expectedUniqueRows == 0) { + return consumed.getUniqueRowCount() == 0 && emptyRounds >= QUIET_ROUNDS_WITHOUT_DATA; + } + return consumed.getUniqueRowCount() >= expectedUniqueRows + && emptyRounds >= QUIET_ROUNDS_AFTER_DATA; + } + + private static boolean hasDrainedExpectedKeys( + final ConsumedRecords consumed, final Set expectedRowKeys, final int emptyRounds) { + return consumed.getRowKeys().containsAll(expectedRowKeys) + && emptyRounds >= QUIET_ROUNDS_AFTER_DATA; + } + + private static String atLeastTimeoutMessage( + final int expectedUniqueRows, final ConsumedRecords consumed) { + return "Expected at least " + + expectedUniqueRows + + " unique row keys before the subscription drain timeout, but collected " + + consumed.getUniqueRowCount() + + ". Consumed records: " + + consumed; + } + + private static String containsTimeoutMessage( + final Set expectedRowKeys, final ConsumedRecords consumed) { + return "Expected row keys were not fully collected before the subscription drain timeout. " + + rowKeyDiffMessage(expectedRowKeys, consumed); + } + + private static String rowKeyDiffMessage( + final Set expectedRowKeys, final ConsumedRecords consumed) { + final Set missingRowKeys = new LinkedHashSet<>(expectedRowKeys); + missingRowKeys.removeAll(consumed.getRowKeys()); + final Set unexpectedRowKeys = new LinkedHashSet<>(consumed.getRowKeys()); + unexpectedRowKeys.removeAll(expectedRowKeys); + return "expected=" + + expectedRowKeys + + ", actual=" + + consumed.getRowKeys() + + ", missing=" + + missingRowKeys + + ", unexpected=" + + unexpectedRowKeys + + ", consumed=" + + consumed; + } + static final class TestIdentifiers { private final String database; @@ -425,6 +589,14 @@ String database(final String suffix) { String topic(final String suffix) { return topic + "_" + suffix; } + + String consumerGroup(final String suffix) { + return consumerGroupId + "_" + suffix; + } + + String consumer(final String suffix) { + return consumerId + "_" + suffix; + } } static final class ConsumedRecords { diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/consensus/local/tablemodel/IoTDBConsensusSubscriptionColumnFilterClusterIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/consensus/local/tablemodel/IoTDBConsensusSubscriptionColumnFilterClusterIT.java new file mode 100644 index 0000000000000..d2fd5552ee75e --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/consensus/local/tablemodel/IoTDBConsensusSubscriptionColumnFilterClusterIT.java @@ -0,0 +1,232 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.consensus.local.tablemodel; + +import org.apache.iotdb.consensus.ConsensusFactory; +import org.apache.iotdb.isession.ITableSession; +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.rpc.subscription.config.TopicConstant; +import org.apache.iotdb.session.subscription.ISubscriptionTableSession; +import org.apache.iotdb.session.subscription.SubscriptionTableSessionBuilder; +import org.apache.iotdb.session.subscription.consumer.table.SubscriptionTablePullConsumer; +import org.apache.iotdb.session.subscription.consumer.table.SubscriptionTablePullConsumerBuilder; +import org.apache.iotdb.subscription.it.AbstractSubscriptionIT; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.util.Collections; +import java.util.LinkedHashSet; +import java.util.Locale; +import java.util.Properties; +import java.util.Set; + +@RunWith(IoTDBTestRunner.class) +@Category({TableClusterIT.class}) +public class IoTDBConsensusSubscriptionColumnFilterClusterIT extends AbstractSubscriptionIT { + + private static final long OWNER_LEASE_DURATION_MS = 60_000L; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + EnvFactory.getEnv() + .getConfig() + .getCommonConfig() + .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS) + .setSchemaReplicationFactor(1) + .setDataReplicationFactor(2) + .setAutoCreateSchemaEnabled(true) + .setSubscriptionEnabled(true) + .setPipeMemoryManagementEnabled(false) + .setIsPipeEnableMemoryCheck(false) + .setSubscriptionOwnerLeaseDurationMsMin(1000); + EnvFactory.getEnv().initClusterEnvironment(1, 3); + } + + @Override + @After + public void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + super.tearDown(); + } + + @Test + public void testAlterColumnFilterRebindsAfterOwnerTransferOnThreeDataNodes() throws Exception { + final ConsensusSubscriptionTableITSupport.TestIdentifiers ids = + ConsensusSubscriptionTableITSupport.newIdentifiers("cluster_owner_rebind"); + final String database = ids.getDatabase(); + final String table = "t1"; + final String schema = "tag1 STRING TAG, s1 INT64 FIELD, s2 DOUBLE FIELD, s3 BOOLEAN FIELD"; + SubscriptionTablePullConsumer ownerConsumer = null; + + try { + ConsensusSubscriptionTableITSupport.createDatabaseAndTable(database, table, schema); + ConsensusSubscriptionTableITSupport.insertRows(database, table, 0L, 1, true, true, true); + createOwnedConsensusTopic(ids.getTopic(), database, table, "column_name = \"s1\""); + + ownerConsumer = createOwnerConsumer(ids.consumer("owner1"), ids.consumerGroup("owner"), 1L); + ownerConsumer.subscribe(ids.getTopic()); + + final Set rowsBeforeTransfer = + ConsensusSubscriptionTableITSupport.insertRows( + database, table, 100L, 3, true, true, true); + final ConsensusSubscriptionTableITSupport.ConsumedRecords beforeTransfer = + ConsensusSubscriptionTableITSupport.pollAndCommitUntilAtLeast( + ownerConsumer, rowsBeforeTransfer.size(), 60); + + ConsensusSubscriptionTableITSupport.assertExactRowKeys(rowsBeforeTransfer, beforeTransfer); + Assert.assertEquals( + Collections.singleton( + ConsensusSubscriptionTableITSupport.normalizeColumnSignature("tag1", "s1")), + beforeTransfer.getSeenColumnSignatures()); + + ownerConsumer.unsubscribe(ids.getTopic()); + ownerConsumer.close(); + ownerConsumer = null; + + addColumn(database, table, "s4 INT32 FIELD"); + transferOwner(ids.getTopic(), "owner2", 2L); + ConsensusSubscriptionTableITSupport.alterConsensusTopicColumnFilter( + ids.getTopic(), "category = \"FIELD\""); + + ownerConsumer = createOwnerConsumer(ids.consumer("owner2"), ids.consumerGroup("owner"), 2L); + ownerConsumer.subscribe(ids.getTopic()); + + final Set rowsAfterTransfer = insertRowsWithS4(database, table, 200L, 3); + final ConsensusSubscriptionTableITSupport.ConsumedRecords afterTransfer = + ConsensusSubscriptionTableITSupport.pollAndCommitUntilContains( + ownerConsumer, rowsAfterTransfer, 80); + + Assert.assertTrue( + "Missing post-transfer rows. Consumed records: " + afterTransfer, + afterTransfer.getRowKeys().containsAll(rowsAfterTransfer)); + Assert.assertTrue( + afterTransfer + .getSeenColumnSignatures() + .contains( + ConsensusSubscriptionTableITSupport.normalizeColumnSignature( + "tag1", "s1", "s2", "s3", "s4"))); + } finally { + ConsensusSubscriptionTableITSupport.cleanup(ownerConsumer, ids.getTopic(), database); + } + } + + private static void createOwnedConsensusTopic( + final String topicName, final String database, final String table, final String columnFilter) + throws Exception { + try (final ISubscriptionTableSession session = + new SubscriptionTableSessionBuilder() + .host(EnvFactory.getEnv().getIP()) + .port(Integer.parseInt(EnvFactory.getEnv().getPort())) + .build()) { + session.open(); + session.dropTopicIfExists(topicName); + + final Properties config = new Properties(); + config.put(TopicConstant.MODE_KEY, TopicConstant.MODE_CONSENSUS_VALUE); + config.put(TopicConstant.FORMAT_KEY, TopicConstant.FORMAT_SESSION_DATA_SETS_HANDLER_VALUE); + config.put(TopicConstant.DATABASE_KEY, database); + config.put(TopicConstant.TABLE_KEY, table); + config.put(TopicConstant.COLUMN_FILTER_KEY, columnFilter); + config.put(TopicConstant.OWNER_ID_KEY, "owner1"); + config.put(TopicConstant.OWNER_EPOCH_KEY, "1"); + config.put( + TopicConstant.OWNER_LEASE_DURATION_MS_KEY, String.valueOf(OWNER_LEASE_DURATION_MS)); + session.createTopic(topicName, config); + } + } + + private static SubscriptionTablePullConsumer createOwnerConsumer( + final String consumerId, final String consumerGroupId, final long ownerEpoch) + throws Exception { + final SubscriptionTablePullConsumer consumer = + (SubscriptionTablePullConsumer) + new SubscriptionTablePullConsumerBuilder() + .host(EnvFactory.getEnv().getIP()) + .port(Integer.parseInt(EnvFactory.getEnv().getPort())) + .consumerId(consumerId) + .consumerGroupId(consumerGroupId) + .ownerId(ownerEpoch == 1L ? "owner1" : "owner2") + .ownerEpoch(ownerEpoch) + .heartbeatIntervalMs(1000) + .endpointsSyncIntervalMs(5000) + .autoCommit(false) + .build(); + consumer.open(); + return consumer; + } + + private static void transferOwner( + final String topicName, final String ownerId, final long ownerEpoch) throws Exception { + try (final ISubscriptionTableSession session = + new SubscriptionTableSessionBuilder() + .host(EnvFactory.getEnv().getIP()) + .port(Integer.parseInt(EnvFactory.getEnv().getPort())) + .build()) { + session.open(); + session.alterTopicOwner(topicName, ownerId, ownerEpoch, OWNER_LEASE_DURATION_MS); + } + } + + private static void addColumn(final String database, final String table, final String column) + throws Exception { + try (final ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + session.executeNonQueryStatement("use " + database); + session.executeNonQueryStatement("alter table " + table + " add column " + column); + } + } + + private static Set insertRowsWithS4( + final String database, final String table, final long startTimestamp, final int rowCount) + throws Exception { + final Set rowKeys = new LinkedHashSet<>(); + try (final ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + session.executeNonQueryStatement("use " + database); + for (int row = 0; row < rowCount; row++) { + final long timestamp = startTimestamp + row; + session.executeNonQueryStatement( + String.format( + Locale.ROOT, + "insert into %s(tag1, s1, s2, s3, s4, time) " + + "values ('tag_%d', %d, %.1f, %s, %d, %d)", + table, + timestamp, + timestamp * 10L, + timestamp + 0.5d, + timestamp % 2 == 0 ? "true" : "false", + timestamp, + timestamp)); + rowKeys.add(ConsensusSubscriptionTableITSupport.rowKey(database, table, timestamp)); + } + session.executeNonQueryStatement("flush"); + } + return rowKeys; + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/consensus/local/tablemodel/IoTDBConsensusSubscriptionFilterTableIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/consensus/local/tablemodel/IoTDBConsensusSubscriptionFilterTableIT.java index d86651f6eea0f..e51b7abbdc69a 100644 --- a/integration-test/src/test/java/org/apache/iotdb/subscription/it/consensus/local/tablemodel/IoTDBConsensusSubscriptionFilterTableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/consensus/local/tablemodel/IoTDBConsensusSubscriptionFilterTableIT.java @@ -154,9 +154,8 @@ public void testColumnFiltering() throws Exception { final String table = "t1"; final String schema = "tag1 STRING TAG, s1 INT64 FIELD, s2 DOUBLE FIELD, s3 BOOLEAN FIELD"; final String expectedColumnSignature = - ConsensusSubscriptionTableITSupport.normalizeColumnSignature("time", "tag1", "s2"); - final Set expectedSeenColumns = - new LinkedHashSet<>(Arrays.asList("time", "tag1", "s2")); + ConsensusSubscriptionTableITSupport.normalizeColumnSignature("tag1", "s2"); + final Set expectedSeenColumns = new LinkedHashSet<>(Arrays.asList("tag1", "s2")); final Set expectedRowKeys = new LinkedHashSet<>(); SubscriptionTablePullConsumer consumer = null; @@ -172,7 +171,7 @@ public void testColumnFiltering() throws Exception { } ConsensusSubscriptionTableITSupport.createConsensusTopic( - ids.getTopic(), database, table, "(tag1|s2)"); + ids.getTopic(), database, table, "column_name REGEXP \"(tag1|s2)\""); consumer = ConsensusSubscriptionTableITSupport.createConsumer( @@ -222,6 +221,8 @@ public void testColumnFilteringWithNoMatchedColumnsReturnsNothing() throws Excep final String database = ids.getDatabase(); final String table = "t1"; final String schema = "tag1 STRING TAG, s1 INT64 FIELD, s2 DOUBLE FIELD"; + final String expectedColumnSignature = + ConsensusSubscriptionTableITSupport.normalizeColumnSignature("tag1", "s2"); SubscriptionTablePullConsumer consumer = null; try { @@ -234,7 +235,7 @@ public void testColumnFilteringWithNoMatchedColumnsReturnsNothing() throws Excep } ConsensusSubscriptionTableITSupport.createConsensusTopic( - ids.getTopic(), database, table, "not_exist"); + ids.getTopic(), database, table, "column_name = \"not_exist\""); consumer = ConsensusSubscriptionTableITSupport.createConsumer( @@ -264,9 +265,196 @@ public void testColumnFilteringWithNoMatchedColumnsReturnsNothing() throws Excep Assert.assertTrue(consumed.getRowKeys().isEmpty()); Assert.assertTrue(consumed.getSeenColumns().isEmpty()); Assert.assertTrue(consumed.getSeenColumnSignatures().isEmpty()); + + ConsensusSubscriptionTableITSupport.alterConsensusTopicColumnFilter( + ids.getTopic(), "column_name = \"s2\""); + final Set rowsAfterEmptyWindow = + ConsensusSubscriptionTableITSupport.insertRows( + database, table, 300L, 3, true, false, true); + final ConsensusSubscriptionTableITSupport.ConsumedRecords consumedAfterEmptyWindow = + ConsensusSubscriptionTableITSupport.pollAndCommitUntilContains( + consumer, rowsAfterEmptyWindow, 50); + + ConsensusSubscriptionTableITSupport.assertExactRowKeys( + rowsAfterEmptyWindow, consumedAfterEmptyWindow); + Assert.assertEquals( + Collections.singleton(expectedColumnSignature), + consumedAfterEmptyWindow.getSeenColumnSignatures()); + ConsensusSubscriptionTableITSupport.assertNoMoreMessages(consumer, 3, Duration.ofMillis(500)); + } finally { + ConsensusSubscriptionTableITSupport.cleanup(consumer, ids.getTopic(), database); + } + } + + @Test + public void testColumnFilteringComplexOperatorsInConsensusQueue() throws Exception { + final ConsensusSubscriptionTableITSupport.TestIdentifiers ids = + ConsensusSubscriptionTableITSupport.newIdentifiers("table_filter_complex_operators"); + final String database = ids.getDatabase(); + final String table = "t1"; + final String schema = "tag1 STRING TAG, s1 INT64 FIELD, s2 DOUBLE FIELD, s3 BOOLEAN FIELD"; + final String expectedColumnSignature = + ConsensusSubscriptionTableITSupport.normalizeColumnSignature("tag1", "s2"); + final String columnFilter = + "datatype IS NOT NULL" + + " AND column_name != \"s1\"" + + " AND column_name NOT IN (\"s1\", \"s3\")" + + " AND column_name NOT LIKE \"unknown%\"" + + " AND column_name NOT REGEXP \"unknown.*\"" + + " AND (column_name LIKE \"s%\" OR column_name = \"tag1\")"; + SubscriptionTablePullConsumer consumer = null; + + try { + ConsensusSubscriptionTableITSupport.createDatabaseAndTable(database, table, schema); + ConsensusSubscriptionTableITSupport.insertRows(database, table, 0L, 1, true, true, true); + + ConsensusSubscriptionTableITSupport.createConsensusTopic( + ids.getTopic(), database, table, columnFilter); + + consumer = + ConsensusSubscriptionTableITSupport.createConsumer( + ids.getConsumerId(), ids.getConsumerGroupId()); + consumer.subscribe(ids.getTopic()); + + final Set expectedRows = + ConsensusSubscriptionTableITSupport.insertRows( + database, table, 250L, 5, true, true, true); + final ConsensusSubscriptionTableITSupport.ConsumedRecords consumed = + ConsensusSubscriptionTableITSupport.pollAndCommitUntilAtLeast( + consumer, expectedRows.size(), 50); + + ConsensusSubscriptionTableITSupport.assertExactRowKeys(expectedRows, consumed); + Assert.assertEquals( + Collections.singleton(expectedColumnSignature), consumed.getSeenColumnSignatures()); + Assert.assertFalse(consumed.getSeenColumns().contains("s1")); + Assert.assertFalse(consumed.getSeenColumns().contains("s3")); ConsensusSubscriptionTableITSupport.assertNoMoreMessages(consumer, 3, Duration.ofMillis(500)); } finally { ConsensusSubscriptionTableITSupport.cleanup(consumer, ids.getTopic(), database); } } + + @Test + public void testColumnFilteringAlterRebindsConsensusQueue() throws Exception { + final ConsensusSubscriptionTableITSupport.TestIdentifiers ids = + ConsensusSubscriptionTableITSupport.newIdentifiers("table_filter_alter_rebind"); + final String database = ids.getDatabase(); + final String table = "t1"; + final String schema = "tag1 STRING TAG, s1 INT64 FIELD, s2 DOUBLE FIELD, s3 BOOLEAN FIELD"; + final String tagAndS1Signature = + ConsensusSubscriptionTableITSupport.normalizeColumnSignature("tag1", "s1"); + final String tagAndS2Signature = + ConsensusSubscriptionTableITSupport.normalizeColumnSignature("tag1", "s2"); + SubscriptionTablePullConsumer consumer = null; + + try { + ConsensusSubscriptionTableITSupport.createDatabaseAndTable(database, table, schema); + ConsensusSubscriptionTableITSupport.insertRows(database, table, 0L, 1, true, true, true); + + ConsensusSubscriptionTableITSupport.createConsensusTopic( + ids.getTopic(), database, table, "column_name = \"s1\""); + + consumer = + ConsensusSubscriptionTableITSupport.createConsumer( + ids.getConsumerId(), ids.getConsumerGroupId()); + consumer.subscribe(ids.getTopic()); + + final Set beforeAlterRows = + ConsensusSubscriptionTableITSupport.insertRows( + database, table, 100L, 5, true, true, true); + final ConsensusSubscriptionTableITSupport.ConsumedRecords beforeAlterConsumed = + ConsensusSubscriptionTableITSupport.pollAndCommitUntilAtLeast( + consumer, beforeAlterRows.size(), 50); + + ConsensusSubscriptionTableITSupport.assertExactRowKeys(beforeAlterRows, beforeAlterConsumed); + Assert.assertEquals( + Collections.singleton(tagAndS1Signature), beforeAlterConsumed.getSeenColumnSignatures()); + ConsensusSubscriptionTableITSupport.assertNoMoreMessages(consumer, 3, Duration.ofMillis(500)); + + ConsensusSubscriptionTableITSupport.alterConsensusTopicColumnFilter( + ids.getTopic(), "column_name = \"s2\""); + + final Set afterAlterRows = + ConsensusSubscriptionTableITSupport.insertRows( + database, table, 200L, 5, true, true, true); + final ConsensusSubscriptionTableITSupport.ConsumedRecords afterAlterConsumed = + ConsensusSubscriptionTableITSupport.pollAndCommitUntilContains( + consumer, afterAlterRows, 60); + + ConsensusSubscriptionTableITSupport.assertExactRowKeys(afterAlterRows, afterAlterConsumed); + Assert.assertEquals( + Collections.singleton(tagAndS2Signature), afterAlterConsumed.getSeenColumnSignatures()); + Assert.assertFalse(afterAlterConsumed.getSeenColumns().contains("s1")); + Assert.assertFalse(afterAlterConsumed.getSeenColumns().contains("s3")); + ConsensusSubscriptionTableITSupport.assertNoMoreMessages(consumer, 3, Duration.ofMillis(500)); + } finally { + ConsensusSubscriptionTableITSupport.cleanup(consumer, ids.getTopic(), database); + } + } + + @Test + public void testColumnFilteringConsistentAcrossConsumerGroups() throws Exception { + final ConsensusSubscriptionTableITSupport.TestIdentifiers ids = + ConsensusSubscriptionTableITSupport.newIdentifiers("table_filter_multi_group"); + final String database = ids.getDatabase(); + final String table = "t1"; + final String schema = "tag1 STRING TAG, s1 INT64 FIELD, s2 DOUBLE FIELD, s3 BOOLEAN FIELD"; + final String expectedColumnSignature = + ConsensusSubscriptionTableITSupport.normalizeColumnSignature("tag1", "s2"); + SubscriptionTablePullConsumer consumer1 = null; + SubscriptionTablePullConsumer consumer2 = null; + + try { + ConsensusSubscriptionTableITSupport.createDatabaseAndTable(database, table, schema); + ConsensusSubscriptionTableITSupport.insertRows(database, table, 0L, 1, true, true, true); + + ConsensusSubscriptionTableITSupport.createConsensusTopic( + ids.getTopic(), database, table, "column_name = \"s2\""); + + consumer1 = + ConsensusSubscriptionTableITSupport.createConsumer( + ids.consumer("g1"), ids.consumerGroup("g1")); + consumer2 = + ConsensusSubscriptionTableITSupport.createConsumer( + ids.consumer("g2"), ids.consumerGroup("g2")); + consumer1.subscribe(ids.getTopic()); + consumer2.subscribe(ids.getTopic()); + + final Set expectedRows = + ConsensusSubscriptionTableITSupport.insertRows( + database, table, 300L, 6, true, true, true); + final ConsensusSubscriptionTableITSupport.ConsumedRecords consumed1 = + ConsensusSubscriptionTableITSupport.pollAndCommitUntilAtLeast( + consumer1, expectedRows.size(), 50); + final ConsensusSubscriptionTableITSupport.ConsumedRecords consumed2 = + ConsensusSubscriptionTableITSupport.pollAndCommitUntilAtLeast( + consumer2, expectedRows.size(), 50); + + ConsensusSubscriptionTableITSupport.assertExactRowKeys(expectedRows, consumed1); + ConsensusSubscriptionTableITSupport.assertExactRowKeys(expectedRows, consumed2); + Assert.assertEquals(consumed1.getRowKeys(), consumed2.getRowKeys()); + Assert.assertEquals( + Collections.singleton(expectedColumnSignature), consumed1.getSeenColumnSignatures()); + Assert.assertEquals( + Collections.singleton(expectedColumnSignature), consumed2.getSeenColumnSignatures()); + ConsensusSubscriptionTableITSupport.assertNoMoreMessages( + consumer1, 3, Duration.ofMillis(500)); + ConsensusSubscriptionTableITSupport.assertNoMoreMessages( + consumer2, 3, Duration.ofMillis(500)); + } finally { + if (consumer2 != null) { + try { + consumer2.unsubscribe(Collections.singleton(ids.getTopic())); + } catch (final Exception ignored) { + // ignored on cleanup + } + try { + consumer2.close(); + } catch (final Exception ignored) { + // ignored on cleanup + } + } + ConsensusSubscriptionTableITSupport.cleanup(consumer1, ids.getTopic(), database); + } + } } diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/tablemodel/IoTDBSubscriptionColumnFilterIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/tablemodel/IoTDBSubscriptionColumnFilterIT.java new file mode 100644 index 0000000000000..14b3bd4bea5ca --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/tablemodel/IoTDBSubscriptionColumnFilterIT.java @@ -0,0 +1,1523 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.dual.tablemodel; + +import org.apache.iotdb.commons.schema.table.TreeViewSchema; +import org.apache.iotdb.commons.schema.table.TsTable; +import org.apache.iotdb.commons.schema.table.column.FieldColumnSchema; +import org.apache.iotdb.commons.schema.table.column.TagColumnSchema; +import org.apache.iotdb.commons.schema.table.column.TimeColumnSchema; +import org.apache.iotdb.db.it.utils.TestUtils; +import org.apache.iotdb.db.subscription.columnfilter.BoundColumnFilter; +import org.apache.iotdb.db.subscription.columnfilter.ColumnFilterBinder; +import org.apache.iotdb.db.subscription.columnfilter.ColumnFilterMatcher; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionTableArchVerification; +import org.apache.iotdb.itbase.env.BaseEnv; +import org.apache.iotdb.rpc.subscription.config.TopicConfig; +import org.apache.iotdb.rpc.subscription.config.TopicConstant; +import org.apache.iotdb.session.subscription.ISubscriptionTableSession; +import org.apache.iotdb.session.subscription.SubscriptionTableSessionBuilder; +import org.apache.iotdb.session.subscription.consumer.ISubscriptionTablePullConsumer; +import org.apache.iotdb.session.subscription.consumer.table.SubscriptionTablePullConsumerBuilder; +import org.apache.iotdb.session.subscription.payload.SubscriptionMessage; +import org.apache.iotdb.session.subscription.payload.SubscriptionMessageType; +import org.apache.iotdb.session.subscription.payload.SubscriptionRecordHandler; +import org.apache.iotdb.session.subscription.payload.SubscriptionTsFileHandler; +import org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant; +import org.apache.iotdb.subscription.it.dual.AbstractSubscriptionDualIT; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.RowRecord; +import org.apache.tsfile.read.query.dataset.ResultSet; +import org.junit.Assert; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.Statement; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; +import java.util.Properties; +import java.util.Set; + +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionTableArchVerification.class}) +public class IoTDBSubscriptionColumnFilterIT extends AbstractSubscriptionDualIT { + + private static final String TABLE_NAME = "t_column_filter"; + private static final String TABLE_SCHEMA = + "tag1 STRING TAG, s1 INT64 FIELD, s2 DOUBLE FIELD, s3 BOOLEAN FIELD"; + private static final String COLUMN_FILTER = "column_name IN (\"time\", \"tag1\", \"s2\")"; + private static final String FIELD_CATEGORY_FILTER = "category = \"FIELD\""; + private static final String FIELD_CATEGORY_REBIND_FILTER = "category IN (\"FIELD\")"; + private static final String MIXED_COLUMN_FILTER = + "( CATEGORY = \"field\" AND datatype IN (\"int64\", \"double\") )" + + " OR \"column_name\" REGEXP \"tag.*\""; + private static final String COMPLEX_OPERATOR_FILTER = + "datatype IS NOT NULL" + + " AND column_name != \"s1\"" + + " AND column_name NOT IN (\"s1\", \"s3\")" + + " AND column_name NOT LIKE \"unknown%\"" + + " AND column_name NOT REGEXP \"unknown.*\"" + + " AND (column_name LIKE \"s%\" OR column_name = \"tag1\")"; + private static final Set EXPECTED_COLUMNS = + new LinkedHashSet<>(Arrays.asList("time", "tag1", "s2")); + private static final Set EXPECTED_ALL_COLUMNS = + new LinkedHashSet<>(Arrays.asList("time", "tag1", "s1", "s2", "s3")); + private static final Set EXPECTED_MIXED_COLUMNS = + new LinkedHashSet<>(Arrays.asList("tag1", "s1", "s2")); + private static final Set EXPECTED_COMPLEX_OPERATOR_COLUMNS = + new LinkedHashSet<>(Arrays.asList("tag1", "s2")); + + @Override + protected void setUpConfig() { + super.setUpConfig(); + senderEnv + .getConfig() + .getCommonConfig() + .setPipeHeartbeatIntervalSecondsForCollectingPipeMeta(30); + senderEnv + .getConfig() + .getCommonConfig() + .setPipeMetaSyncerInitialSyncDelayMinutes(1) + .setPipeMemoryManagementEnabled(false) + .setIsPipeEnableMemoryCheck(false); + senderEnv + .getConfig() + .getCommonConfig() + .setPipeMetaSyncerSyncIntervalMinutes(1) + .setPipeMemoryManagementEnabled(false) + .setIsPipeEnableMemoryCheck(false); + } + + @Test + public void testLiveRecordHandlerColumnFilter() throws Exception { + final String database = databaseName("record"); + final String topicName = topicName("record"); + final String consumerId = consumerName("record"); + final String consumerGroupId = consumerGroupName("record"); + + try { + createDatabaseAndTable(senderEnv, database, TABLE_NAME, TABLE_SCHEMA); + createTopic( + topicName, + database, + TABLE_NAME, + TopicConstant.FORMAT_RECORD_HANDLER_VALUE, + COLUMN_FILTER); + + try (final ISubscriptionTablePullConsumer consumer = + createConsumer(consumerId, consumerGroupId)) { + consumer.subscribe(topicName); + insertRows(senderEnv, database, TABLE_NAME, 1, 4); + + final Set expectedTimestamps = new LinkedHashSet<>(Arrays.asList(1L, 2L, 3L)); + final ConsumedRecordStats stats = + pollRecordMessagesForTimestamps(consumer, expectedTimestamps, true); + + Assert.assertEquals(expectedTimestamps, stats.timestamps); + Assert.assertEquals(EXPECTED_COLUMNS, stats.columnNames); + Assert.assertFalse(stats.columnNames.contains("s1")); + Assert.assertFalse(stats.columnNames.contains("s3")); + } + } finally { + cleanup(topicName, database); + } + } + + @Test + public void testLiveRecordHandlerWithoutColumnFilterPreservesAllColumns() throws Exception { + final String database = databaseName("default"); + final String topicName = topicName("default"); + final String consumerId = consumerName("default"); + final String consumerGroupId = consumerGroupName("default"); + + try { + createDatabaseAndTable(senderEnv, database, TABLE_NAME, TABLE_SCHEMA); + createTopicWithoutColumnFilter( + topicName, database, TABLE_NAME, TopicConstant.FORMAT_RECORD_HANDLER_VALUE); + + try (final ISubscriptionTablePullConsumer consumer = + createConsumer(consumerId, consumerGroupId)) { + consumer.subscribe(topicName); + insertRows(senderEnv, database, TABLE_NAME, 110, 113); + + final Set expectedTimestamps = new LinkedHashSet<>(Arrays.asList(110L, 111L, 112L)); + final ConsumedRecordStats stats = + pollRecordMessagesForTimestamps(consumer, expectedTimestamps, true); + + Assert.assertEquals(expectedTimestamps, stats.timestamps); + Assert.assertEquals(EXPECTED_ALL_COLUMNS, stats.columnNames); + } + } finally { + cleanup(topicName, database); + } + } + + @Test + public void testLiveRecordHandlerColumnFilterKeepsHistoricalAndRealtimeConsistent() + throws Exception { + final String database = databaseName("live_both"); + final String topicName = topicName("live_both"); + final String consumerId = consumerName("live_both"); + final String consumerGroupId = consumerGroupName("live_both"); + + try { + createDatabaseAndTable(senderEnv, database, TABLE_NAME, TABLE_SCHEMA); + insertRows(senderEnv, database, TABLE_NAME, 180, 183); + createTopic( + topicName, + database, + TABLE_NAME, + TopicConstant.FORMAT_RECORD_HANDLER_VALUE, + COLUMN_FILTER); + + try (final ISubscriptionTablePullConsumer consumer = + createConsumer(consumerId, consumerGroupId)) { + consumer.subscribe(topicName); + + final Set historicalTimestamps = new LinkedHashSet<>(Arrays.asList(180L, 181L, 182L)); + final ConsumedRecordStats historicalStats = + pollRecordMessagesForTimestamps(consumer, historicalTimestamps, true); + Assert.assertEquals(EXPECTED_COLUMNS, historicalStats.columnNames); + + insertRows(senderEnv, database, TABLE_NAME, 190, 193); + + final Set realtimeTimestamps = new LinkedHashSet<>(Arrays.asList(190L, 191L, 192L)); + final ConsumedRecordStats realtimeStats = + pollRecordMessagesForTimestamps(consumer, realtimeTimestamps, true); + Assert.assertEquals(EXPECTED_COLUMNS, realtimeStats.columnNames); + } + } finally { + cleanup(topicName, database); + } + } + + @Test + public void testLiveRecordHandlerColumnFilterAppliesToMultipleConsumerGroups() throws Exception { + final String database = databaseName("multi_group"); + final String topicName = topicName("multi_group"); + final String firstConsumerId = consumerName("multi_group_a"); + final String firstConsumerGroupId = consumerGroupName("multi_group_a"); + final String secondConsumerId = consumerName("multi_group_b"); + final String secondConsumerGroupId = consumerGroupName("multi_group_b"); + + try { + createDatabaseAndTable(senderEnv, database, TABLE_NAME, TABLE_SCHEMA); + createTopic( + topicName, + database, + TABLE_NAME, + TopicConstant.FORMAT_RECORD_HANDLER_VALUE, + COLUMN_FILTER); + + try (final ISubscriptionTablePullConsumer firstConsumer = + createConsumer(firstConsumerId, firstConsumerGroupId); + final ISubscriptionTablePullConsumer secondConsumer = + createConsumer(secondConsumerId, secondConsumerGroupId)) { + firstConsumer.subscribe(topicName); + secondConsumer.subscribe(topicName); + insertRows(senderEnv, database, TABLE_NAME, 170, 173); + + final Set expectedTimestamps = new LinkedHashSet<>(Arrays.asList(170L, 171L, 172L)); + final ConsumedRecordStats firstStats = + pollRecordMessagesForTimestamps(firstConsumer, expectedTimestamps, true); + final ConsumedRecordStats secondStats = + pollRecordMessagesForTimestamps(secondConsumer, expectedTimestamps, true); + + Assert.assertEquals(EXPECTED_COLUMNS, firstStats.columnNames); + Assert.assertEquals(EXPECTED_COLUMNS, secondStats.columnNames); + } + } finally { + cleanup(topicName, database); + } + } + + @Test + public void testSnapshotRecordHandlerColumnFilter() throws Exception { + final String database = databaseName("snapshot"); + final String topicName = topicName("snapshot"); + final String consumerId = consumerName("snapshot"); + final String consumerGroupId = consumerGroupName("snapshot"); + + try { + createDatabaseAndTable(senderEnv, database, TABLE_NAME, TABLE_SCHEMA); + insertRows(senderEnv, database, TABLE_NAME, 20, 23); + createTopic( + topicName, + database, + TABLE_NAME, + TopicConstant.MODE_SNAPSHOT_VALUE, + TopicConstant.FORMAT_RECORD_HANDLER_VALUE, + COLUMN_FILTER); + + try (final ISubscriptionTablePullConsumer consumer = + createConsumer(consumerId, consumerGroupId)) { + consumer.subscribe(topicName); + + final Set expectedTimestamps = new LinkedHashSet<>(Arrays.asList(20L, 21L, 22L)); + final ConsumedRecordStats stats = + pollRecordMessagesForTimestamps(consumer, expectedTimestamps, true); + + Assert.assertEquals(expectedTimestamps, stats.timestamps); + Assert.assertEquals(EXPECTED_COLUMNS, stats.columnNames); + } + } finally { + cleanup(topicName, database); + } + } + + @Test + public void testLiveRecordHandlerMixedCaseInsensitiveColumnFilter() throws Exception { + final String database = databaseName("mixed"); + final String topicName = topicName("mixed"); + final String consumerId = consumerName("mixed"); + final String consumerGroupId = consumerGroupName("mixed"); + + try { + createDatabaseAndTable(senderEnv, database, TABLE_NAME, TABLE_SCHEMA); + createTopic( + topicName, + database, + TABLE_NAME, + TopicConstant.FORMAT_RECORD_HANDLER_VALUE, + MIXED_COLUMN_FILTER); + + try (final ISubscriptionTablePullConsumer consumer = + createConsumer(consumerId, consumerGroupId)) { + consumer.subscribe(topicName); + insertRows(senderEnv, database, TABLE_NAME, 30, 33); + + final Set expectedTimestamps = new LinkedHashSet<>(Arrays.asList(30L, 31L, 32L)); + final ConsumedRecordStats stats = + pollRecordMessagesForTimestamps(consumer, expectedTimestamps, false); + + Assert.assertEquals(expectedTimestamps, stats.timestamps); + Assert.assertEquals(EXPECTED_MIXED_COLUMNS, stats.columnNames); + Assert.assertFalse(stats.columnNames.contains("time")); + Assert.assertFalse(stats.columnNames.contains("s3")); + } + } finally { + cleanup(topicName, database); + } + } + + @Test + public void testLiveRecordHandlerComplexOperatorColumnFilter() throws Exception { + final String database = databaseName("operators"); + final String topicName = topicName("operators"); + final String consumerId = consumerName("operators"); + final String consumerGroupId = consumerGroupName("operators"); + + try { + createDatabaseAndTable(senderEnv, database, TABLE_NAME, TABLE_SCHEMA); + createTopic( + topicName, + database, + TABLE_NAME, + TopicConstant.FORMAT_RECORD_HANDLER_VALUE, + COMPLEX_OPERATOR_FILTER); + + try (final ISubscriptionTablePullConsumer consumer = + createConsumer(consumerId, consumerGroupId)) { + consumer.subscribe(topicName); + insertRows(senderEnv, database, TABLE_NAME, 100, 103); + + final Set expectedTimestamps = new LinkedHashSet<>(Arrays.asList(100L, 101L, 102L)); + final ConsumedRecordStats stats = + pollRecordMessagesForTimestamps(consumer, expectedTimestamps, false); + + Assert.assertEquals(expectedTimestamps, stats.timestamps); + Assert.assertEquals(EXPECTED_COMPLEX_OPERATOR_COLUMNS, stats.columnNames); + Assert.assertFalse(stats.columnNames.contains("time")); + Assert.assertFalse(stats.columnNames.contains("s1")); + Assert.assertFalse(stats.columnNames.contains("s3")); + } + } finally { + cleanup(topicName, database); + } + } + + @Test + public void testLiveRecordHandlerColumnFilterDropsMatchedAttributeColumn() throws Exception { + final String database = databaseName("attribute"); + final String topicName = topicName("attribute"); + final String consumerId = consumerName("attribute"); + final String consumerGroupId = consumerGroupName("attribute"); + final String schemaWithAttribute = + "tag1 STRING TAG, attr1 STRING ATTRIBUTE, s1 INT64 FIELD, s2 DOUBLE FIELD"; + + try { + createDatabaseAndTable(senderEnv, database, TABLE_NAME, schemaWithAttribute); + createTopic( + topicName, + database, + TABLE_NAME, + TopicConstant.FORMAT_RECORD_HANDLER_VALUE, + "category = \"ATTRIBUTE\" OR column_name = \"s2\""); + + try (final ISubscriptionTablePullConsumer consumer = + createConsumer(consumerId, consumerGroupId)) { + consumer.subscribe(topicName); + insertRowsWithAttribute(senderEnv, database, TABLE_NAME, 160, 163); + + final Set expectedTimestamps = new LinkedHashSet<>(Arrays.asList(160L, 161L, 162L)); + final ConsumedRecordStats stats = + pollRecordMessagesForTimestamps(consumer, expectedTimestamps, false); + + Assert.assertEquals(expectedTimestamps, stats.timestamps); + Assert.assertEquals(new LinkedHashSet<>(Arrays.asList("tag1", "s2")), stats.columnNames); + Assert.assertFalse(stats.columnNames.contains("attr1")); + Assert.assertFalse(stats.columnNames.contains("s1")); + } + } finally { + cleanup(topicName, database); + } + } + + @Test + public void testTreeViewColumnFilterBindingUsesSourceFieldName() throws Exception { + final String database = databaseName("view"); + final String viewName = TABLE_NAME + "_view"; + final Map attributes = new LinkedHashMap<>(); + attributes.put("__system.sql-dialect", "table"); + attributes.put(TopicConstant.DATABASE_KEY, database); + attributes.put(TopicConstant.TABLE_KEY, viewName); + attributes.put(TopicConstant.COLUMN_FILTER_KEY, "column_name = \"s_view\""); + + final BoundColumnFilter boundColumnFilter = + new ColumnFilterBinder() + .bind( + new TopicConfig(attributes), + Map.of(database, Map.of(viewName, createTreeViewSchema(viewName)))); + final ColumnFilterMatcher matcher = + ColumnFilterMatcher.fromBoundColumnFilter(boundColumnFilter); + + Assert.assertTrue(matcher.match(database, viewName, "tag1")); + Assert.assertTrue(matcher.match(database, viewName, "s_src")); + Assert.assertFalse(matcher.match(database, viewName, "s_view")); + } + + @Test + public void testCreateTopicWithTreeViewColumnFilterUsesViewFieldName() throws Exception { + final String database = databaseName("view_topic"); + final String treeDatabase = database + "_tree"; + final String viewName = TABLE_NAME + "_view"; + final String topicName = topicName("view_topic"); + + try { + createTreeView(senderEnv, database, treeDatabase, viewName); + createTopic( + topicName, + database, + viewName, + TopicConstant.FORMAT_RECORD_HANDLER_VALUE, + "column_name = \"s_view\""); + } finally { + cleanup(topicName, database); + dropTreeDatabase(senderEnv, treeDatabase); + } + } + + @Test + public void testLiveTsFileColumnFilter() throws Exception { + final String database = databaseName("tsfile"); + final String topicName = topicName("tsfile"); + final String consumerId = consumerName("tsfile"); + final String consumerGroupId = consumerGroupName("tsfile"); + + try { + createDatabaseAndTable(senderEnv, database, TABLE_NAME, TABLE_SCHEMA); + createDatabaseAndTable(receiverEnv, database, TABLE_NAME, TABLE_SCHEMA); + createTopic( + topicName, database, TABLE_NAME, TopicConstant.FORMAT_TS_FILE_VALUE, COLUMN_FILTER); + + try (final ISubscriptionTablePullConsumer consumer = + createConsumer(consumerId, consumerGroupId)) { + consumer.subscribe(topicName); + insertRows(senderEnv, database, TABLE_NAME, 10, 13); + pollTsFileMessagesAndLoad(consumer, database, 3); + } + + assertLoadedTsFileRows(database, 3); + } finally { + cleanup(topicName, database); + } + } + + @Test + public void testSnapshotTsFileColumnFilter() throws Exception { + final String database = databaseName("snapshot_tsfile"); + final String topicName = topicName("snapshot_tsfile"); + final String consumerId = consumerName("snapshot_tsfile"); + final String consumerGroupId = consumerGroupName("snapshot_tsfile"); + + try { + createDatabaseAndTable(senderEnv, database, TABLE_NAME, TABLE_SCHEMA); + createDatabaseAndTable(receiverEnv, database, TABLE_NAME, TABLE_SCHEMA); + insertRows(senderEnv, database, TABLE_NAME, 200, 203); + createTopic( + topicName, + database, + TABLE_NAME, + TopicConstant.MODE_SNAPSHOT_VALUE, + TopicConstant.FORMAT_TS_FILE_VALUE, + COLUMN_FILTER); + + try (final ISubscriptionTablePullConsumer consumer = + createConsumer(consumerId, consumerGroupId)) { + consumer.subscribe(topicName); + pollTsFileMessagesAndLoad(consumer, database, 3); + } + + assertLoadedTsFileRows(database, 3); + } finally { + cleanup(topicName, database); + } + } + + @Test + public void testLiveTsFileTrueColumnFilterPreservesAllColumns() throws Exception { + final String database = databaseName("tsfile_true"); + final String topicName = topicName("tsfile_true"); + final String consumerId = consumerName("tsfile_true"); + final String consumerGroupId = consumerGroupName("tsfile_true"); + + try { + createDatabaseAndTable(senderEnv, database, TABLE_NAME, TABLE_SCHEMA); + createDatabaseAndTable(receiverEnv, database, TABLE_NAME, TABLE_SCHEMA); + createTopic(topicName, database, TABLE_NAME, TopicConstant.FORMAT_TS_FILE_VALUE, "true"); + + try (final ISubscriptionTablePullConsumer consumer = + createConsumer(consumerId, consumerGroupId)) { + consumer.subscribe(topicName); + insertRows(senderEnv, database, TABLE_NAME, 120, 123); + pollTsFileMessagesAndLoad(consumer, database, 3); + } + + assertLoadedTsFileRowsWithAllColumns(database, 3); + } finally { + cleanup(topicName, database); + } + } + + @Test + public void testLiveRecordHandlerAlterColumnFilterRebindsNewData() throws Exception { + final String database = databaseName("alter"); + final String topicName = topicName("alter"); + final String consumerId = consumerName("alter"); + final String consumerGroupId = consumerGroupName("alter"); + + try { + createDatabaseAndTable(senderEnv, database, TABLE_NAME, TABLE_SCHEMA); + createTopic( + topicName, + database, + TABLE_NAME, + TopicConstant.FORMAT_RECORD_HANDLER_VALUE, + "column_name = \"s1\""); + + try (final ISubscriptionTablePullConsumer consumer = + createConsumer(consumerId, consumerGroupId)) { + consumer.subscribe(topicName); + insertRows(senderEnv, database, TABLE_NAME, 40, 43); + + final Set beforeAlterTimestamps = new LinkedHashSet<>(Arrays.asList(40L, 41L, 42L)); + final ConsumedRecordStats beforeAlter = + pollRecordMessagesForTimestamps(consumer, beforeAlterTimestamps, false); + Assert.assertEquals(beforeAlterTimestamps, beforeAlter.timestamps); + Assert.assertEquals( + new LinkedHashSet<>(Arrays.asList("tag1", "s1")), beforeAlter.columnNames); + + alterTopicColumnFilter(topicName, COLUMN_FILTER); + insertRows(senderEnv, database, TABLE_NAME, 50, 53); + + final Set afterAlterTimestamps = new LinkedHashSet<>(Arrays.asList(50L, 51L, 52L)); + final ConsumedRecordStats afterAlter = + pollRecordMessagesForTimestamps(consumer, afterAlterTimestamps, true); + Assert.assertEquals(afterAlterTimestamps, afterAlter.timestamps); + Assert.assertEquals(EXPECTED_COLUMNS, afterAlter.columnNames); + } + } finally { + cleanup(topicName, database); + } + } + + @Test + public void testLiveRecordHandlerFalseColumnFilterSkipsRowsAndRebinds() throws Exception { + final String database = databaseName("false"); + final String topicName = topicName("false"); + final String consumerId = consumerName("false"); + final String consumerGroupId = consumerGroupName("false"); + + try { + createDatabaseAndTable(senderEnv, database, TABLE_NAME, TABLE_SCHEMA); + createTopic( + topicName, database, TABLE_NAME, TopicConstant.FORMAT_RECORD_HANDLER_VALUE, "false"); + + try (final ISubscriptionTablePullConsumer consumer = + createConsumer(consumerId, consumerGroupId)) { + consumer.subscribe(topicName); + insertRows(senderEnv, database, TABLE_NAME, 60, 63); + + assertNoUserRows(consumer, 5); + + alterTopicColumnFilter(topicName, COLUMN_FILTER); + insertRows(senderEnv, database, TABLE_NAME, 70, 73); + + final Set expectedTimestamps = new LinkedHashSet<>(Arrays.asList(70L, 71L, 72L)); + final ConsumedRecordStats stats = + pollRecordMessagesForTimestamps(consumer, expectedTimestamps, true); + + Assert.assertEquals(EXPECTED_COLUMNS, stats.columnNames); + Assert.assertEquals(expectedTimestamps, stats.timestamps); + } + } finally { + cleanup(topicName, database); + } + } + + @Test + public void testLiveRecordHandlerColumnFilterSurvivesSenderRestart() throws Exception { + final String database = databaseName("restart"); + final String topicName = topicName("restart"); + final String consumerId = consumerName("restart"); + final String consumerGroupId = consumerGroupName("restart"); + + try { + createDatabaseAndTable(senderEnv, database, TABLE_NAME, TABLE_SCHEMA); + createTopic( + topicName, + database, + TABLE_NAME, + TopicConstant.FORMAT_RECORD_HANDLER_VALUE, + COLUMN_FILTER); + + TestUtils.restartCluster(senderEnv); + + try (final ISubscriptionTablePullConsumer consumer = + createConsumer(consumerId, consumerGroupId)) { + consumer.subscribe(topicName); + insertRows(senderEnv, database, TABLE_NAME, 130, 133); + + final Set expectedTimestamps = new LinkedHashSet<>(Arrays.asList(130L, 131L, 132L)); + final ConsumedRecordStats stats = + pollRecordMessagesForTimestamps(consumer, expectedTimestamps, true); + + Assert.assertEquals(expectedTimestamps, stats.timestamps); + Assert.assertEquals(EXPECTED_COLUMNS, stats.columnNames); + Assert.assertFalse(stats.columnNames.contains("s1")); + Assert.assertFalse(stats.columnNames.contains("s3")); + } + } finally { + cleanup(topicName, database); + } + } + + @Test + public void testLiveRecordHandlerRestartResnapshotsNewMatchingTable() throws Exception { + final String database = databaseName("restart_new"); + final String topicName = topicName("restart_new"); + final String consumerId = consumerName("restart_new"); + final String consumerGroupId = consumerGroupName("restart_new"); + final String initialTableName = TABLE_NAME + "_restart_old"; + final String newTableName = TABLE_NAME + "_restart_new"; + + try { + createDatabaseAndTable(senderEnv, database, initialTableName, TABLE_SCHEMA); + createTopic( + topicName, + database, + TABLE_NAME + "_restart_.*", + TopicConstant.FORMAT_RECORD_HANDLER_VALUE, + FIELD_CATEGORY_FILTER); + createTable(senderEnv, database, newTableName, TABLE_SCHEMA); + + TestUtils.restartCluster(senderEnv); + + try (final ISubscriptionTablePullConsumer consumer = + createConsumer(consumerId, consumerGroupId)) { + consumer.subscribe(topicName); + insertRows(senderEnv, database, newTableName, 240, 243); + + final Set expectedTimestamps = new LinkedHashSet<>(Arrays.asList(240L, 241L, 242L)); + final ConsumedRecordStats stats = + pollRecordMessagesForTimestamps(consumer, expectedTimestamps, false); + + Assert.assertEquals(expectedTimestamps, stats.timestamps); + Assert.assertEquals( + new LinkedHashSet<>(Arrays.asList("tag1", "s1", "s2", "s3")), stats.columnNames); + Assert.assertEquals(new LinkedHashSet<>(Arrays.asList(newTableName)), stats.tableNames); + } + } finally { + cleanup(topicName, database); + } + } + + @Test + public void testLiveRecordHandlerStrictSnapshotRequiresAlterAfterAddColumn() throws Exception { + final String database = databaseName("strict"); + final String topicName = topicName("strict"); + final String consumerId = consumerName("strict"); + final String consumerGroupId = consumerGroupName("strict"); + + try { + createDatabaseAndTable(senderEnv, database, TABLE_NAME, TABLE_SCHEMA); + createTopic( + topicName, + database, + TABLE_NAME, + TopicConstant.FORMAT_RECORD_HANDLER_VALUE, + FIELD_CATEGORY_FILTER); + + try (final ISubscriptionTablePullConsumer consumer = + createConsumer(consumerId, consumerGroupId)) { + consumer.subscribe(topicName); + addColumn(senderEnv, database, TABLE_NAME, "s4 INT32 FIELD"); + insertRowsWithS4(senderEnv, database, TABLE_NAME, 80, 84); + + final Set beforeAlterTimestamps = + new LinkedHashSet<>(Arrays.asList(80L, 81L, 82L, 83L)); + final ConsumedRecordStats beforeAlter = + pollRecordMessagesForTimestamps(consumer, beforeAlterTimestamps, false); + Assert.assertTrue(beforeAlter.columnNames.contains("tag1")); + Assert.assertTrue(beforeAlter.columnNames.contains("s1")); + Assert.assertTrue(beforeAlter.columnNames.contains("s2")); + Assert.assertTrue(beforeAlter.columnNames.contains("s3")); + Assert.assertFalse(beforeAlter.columnNames.contains("s4")); + + alterTopicColumnFilter(topicName, FIELD_CATEGORY_REBIND_FILTER); + insertRowsWithS4(senderEnv, database, TABLE_NAME, 90, 94); + + final Set afterAlterTimestamps = + new LinkedHashSet<>(Arrays.asList(90L, 91L, 92L, 93L)); + final ConsumedRecordStats afterAlter = + pollRecordMessagesForTimestamps(consumer, afterAlterTimestamps, false); + Assert.assertTrue(afterAlter.columnNames.contains("tag1")); + Assert.assertTrue(afterAlter.columnNames.contains("s1")); + Assert.assertTrue(afterAlter.columnNames.contains("s2")); + Assert.assertTrue(afterAlter.columnNames.contains("s3")); + Assert.assertTrue(afterAlter.columnNames.contains("s4")); + Assert.assertEquals(afterAlterTimestamps, afterAlter.timestamps); + } + } finally { + cleanup(topicName, database); + } + } + + @Test + public void testLiveRecordHandlerExpressionCoverageAndTagRetention() throws Exception { + final String database = databaseName("expr"); + final String topicName = topicName("expr"); + final String consumerId = consumerName("expr"); + final String consumerGroupId = consumerGroupName("expr"); + final String expressionFilter = + "database IS NOT NULL" + + " AND table_name IS NOT NULL" + + " AND (database IS NULL OR datatype IN (\"DOUBLE\", \"FLOAT\", \"INT64\"))" + + " AND column_name NOT IN (\"s1\", \"s3\")" + + " AND column_name NOT LIKE \"unknown%\"" + + " AND column_name NOT REGEXP \"unknown.*\""; + + try { + createDatabaseAndTable(senderEnv, database, TABLE_NAME, TABLE_SCHEMA); + createTopic( + topicName, + database, + TABLE_NAME, + TopicConstant.FORMAT_RECORD_HANDLER_VALUE, + expressionFilter); + + try (final ISubscriptionTablePullConsumer consumer = + createConsumer(consumerId, consumerGroupId)) { + consumer.subscribe(topicName); + insertRows(senderEnv, database, TABLE_NAME, 250, 253); + + final Set expectedTimestamps = new LinkedHashSet<>(Arrays.asList(250L, 251L, 252L)); + final ConsumedRecordStats stats = + pollRecordMessagesForTimestamps(consumer, expectedTimestamps, false); + + Assert.assertEquals(expectedTimestamps, stats.timestamps); + Assert.assertEquals(new LinkedHashSet<>(Arrays.asList("tag1", "s2")), stats.columnNames); + Assert.assertTrue( + "TAG column must be retained when only FIELD matches", + stats.columnNames.contains("tag1")); + Assert.assertFalse(stats.columnNames.contains("time")); + Assert.assertFalse(stats.columnNames.contains("s1")); + Assert.assertFalse(stats.columnNames.contains("s3")); + } + } finally { + cleanup(topicName, database); + } + } + + @Test + public void testLiveRecordHandlerColumnFilterMatchesCustomTimeColumn() throws Exception { + final String database = databaseName("custom_time"); + final String topicName = topicName("custom_time"); + final String consumerId = consumerName("custom_time"); + final String consumerGroupId = consumerGroupName("custom_time"); + final String customTimeTableSchema = + "event_time TIMESTAMP TIME, tag1 STRING TAG, s1 INT64 FIELD, s2 DOUBLE FIELD"; + + try { + createDatabaseAndTable(senderEnv, database, TABLE_NAME, customTimeTableSchema); + createTopic( + topicName, + database, + TABLE_NAME, + TopicConstant.FORMAT_RECORD_HANDLER_VALUE, + "column_name = \"event_time\" OR column_name = \"s2\""); + + try (final ISubscriptionTablePullConsumer consumer = + createConsumer(consumerId, consumerGroupId)) { + consumer.subscribe(topicName); + insertRowsWithCustomTimeColumn(senderEnv, database, TABLE_NAME, 140, 143); + + final Set expectedTimestamps = new LinkedHashSet<>(Arrays.asList(140L, 141L, 142L)); + final ConsumedRecordStats stats = + pollRecordMessagesForTimestamps(consumer, expectedTimestamps, true); + + Assert.assertEquals(expectedTimestamps, stats.timestamps); + Assert.assertEquals(EXPECTED_COLUMNS, stats.columnNames); + Assert.assertFalse(stats.columnNames.contains("s1")); + } + } finally { + cleanup(topicName, database); + } + } + + @Test + public void testLiveRecordHandlerColumnFilterUsesAlteredDataTypeAtBindTime() throws Exception { + final String database = databaseName("alter_type"); + final String topicName = topicName("alter_type"); + final String consumerId = consumerName("alter_type"); + final String consumerGroupId = consumerGroupName("alter_type"); + final String initialSchema = + "tag1 STRING TAG, s1 INT32 FIELD, s2 DOUBLE FIELD, s3 BOOLEAN FIELD"; + + try { + createDatabaseAndTable(senderEnv, database, TABLE_NAME, initialSchema); + alterColumnType(senderEnv, database, TABLE_NAME, "s1", "INT64"); + createTopic( + topicName, + database, + TABLE_NAME, + TopicConstant.FORMAT_RECORD_HANDLER_VALUE, + "datatype = \"INT64\""); + + try (final ISubscriptionTablePullConsumer consumer = + createConsumer(consumerId, consumerGroupId)) { + consumer.subscribe(topicName); + insertRows(senderEnv, database, TABLE_NAME, 150, 153); + + final Set expectedTimestamps = new LinkedHashSet<>(Arrays.asList(150L, 151L, 152L)); + final ConsumedRecordStats stats = + pollRecordMessagesForTimestamps(consumer, expectedTimestamps, false); + + Assert.assertEquals(expectedTimestamps, stats.timestamps); + Assert.assertEquals(new LinkedHashSet<>(Arrays.asList("tag1", "s1")), stats.columnNames); + Assert.assertFalse(stats.columnNames.contains("time")); + Assert.assertFalse(stats.columnNames.contains("s2")); + Assert.assertFalse(stats.columnNames.contains("s3")); + } + } finally { + cleanup(topicName, database); + } + } + + @Test + public void testLiveRecordHandlerColumnFilterAppliesTimeSelectionPerTable() throws Exception { + final String database = databaseName("multi_time"); + final String topicName = topicName("multi_time"); + final String consumerId = consumerName("multi_time"); + final String consumerGroupId = consumerGroupName("multi_time"); + final String timeSelectedTableName = TABLE_NAME + "_time_selected"; + final String timeUnselectedTableName = TABLE_NAME + "_time_unselected"; + + try { + createDatabaseAndTable(senderEnv, database, timeSelectedTableName, TABLE_SCHEMA); + createTable(senderEnv, database, timeUnselectedTableName, TABLE_SCHEMA); + createTopic( + topicName, + database, + TABLE_NAME + "_time_.*", + TopicConstant.FORMAT_RECORD_HANDLER_VALUE, + "(table_name = \"" + + timeSelectedTableName + + "\" AND column_name = \"time\")" + + " OR (table_name = \"" + + timeUnselectedTableName + + "\" AND column_name = \"s2\")"); + + try (final ISubscriptionTablePullConsumer consumer = + createConsumer(consumerId, consumerGroupId)) { + consumer.subscribe(topicName); + insertRows(senderEnv, database, timeSelectedTableName, 210, 213); + insertRows(senderEnv, database, timeUnselectedTableName, 220, 223); + + final Map expectedTimeSelectedByTimestamp = new LinkedHashMap<>(); + expectedTimeSelectedByTimestamp.put(210L, true); + expectedTimeSelectedByTimestamp.put(211L, true); + expectedTimeSelectedByTimestamp.put(212L, true); + expectedTimeSelectedByTimestamp.put(220L, false); + expectedTimeSelectedByTimestamp.put(221L, false); + expectedTimeSelectedByTimestamp.put(222L, false); + final ConsumedRecordStats stats = + pollRecordMessagesForTimestampTimeSelection(consumer, expectedTimeSelectedByTimestamp); + + Assert.assertEquals(expectedTimeSelectedByTimestamp.keySet(), stats.timestamps); + Assert.assertEquals(expectedTimeSelectedByTimestamp, stats.timeSelectedByTimestamp); + Assert.assertEquals( + new LinkedHashSet<>(Arrays.asList("time", "tag1", "s2")), stats.columnNames); + Assert.assertFalse(stats.columnNames.contains("s1")); + Assert.assertFalse(stats.columnNames.contains("s3")); + } + } finally { + cleanup(topicName, database); + } + } + + @Test + public void testCreateAndAlterRejectInvalidColumnFilter() throws Exception { + final String database = databaseName("invalid"); + final String invalidTopicName = topicName("invalid_create"); + final String validTopicName = topicName("invalid_alter"); + + try { + createDatabaseAndTable(senderEnv, database, TABLE_NAME, TABLE_SCHEMA); + + final Exception createException = + Assert.assertThrows( + Exception.class, + () -> + createTopic( + invalidTopicName, + database, + TABLE_NAME, + TopicConstant.FORMAT_RECORD_HANDLER_VALUE, + "temperature > 10")); + Assert.assertFalse(String.valueOf(createException.getMessage()).contains("tag_")); + + createTopic( + validTopicName, + database, + TABLE_NAME, + TopicConstant.FORMAT_RECORD_HANDLER_VALUE, + COLUMN_FILTER); + final Exception alterException = + Assert.assertThrows( + Exception.class, () -> alterTopicColumnFilter(validTopicName, "owner = \"alice\"")); + Assert.assertFalse(String.valueOf(alterException.getMessage()).contains("tag_")); + } finally { + cleanup(invalidTopicName, database); + cleanup(validTopicName, database); + } + } + + private String databaseName(final String suffix) { + return "cf_" + suffix + "_" + testId(); + } + + private String topicName(final String suffix) { + return "topic_cf_" + suffix + "_" + testId(); + } + + private String consumerName(final String suffix) { + return "consumer_cf_" + suffix + "_" + testId(); + } + + private String consumerGroupName(final String suffix) { + return "cg_cf_" + suffix + "_" + testId(); + } + + private String testId() { + return Integer.toUnsignedString(testName.getDisplayName().hashCode(), 36); + } + + private static void createDatabaseAndTable( + final BaseEnv env, final String database, final String tableName, final String schema) + throws Exception { + try (final Connection connection = env.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute("create database " + database); + statement.execute("use " + database); + statement.execute(String.format("create table %s (%s)", tableName, schema)); + } + } + + private static void createTable( + final BaseEnv env, final String database, final String tableName, final String schema) + throws Exception { + try (final Connection connection = env.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute("use " + database); + statement.execute(String.format("create table %s (%s)", tableName, schema)); + } + } + + private static TsTable createTreeViewSchema(final String viewName) { + final TsTable table = new TsTable(viewName); + table.addProp(TreeViewSchema.TREE_PATH_PATTERN, "root.view.**"); + table.addColumnSchema(new TimeColumnSchema("time", TSDataType.TIMESTAMP)); + table.addColumnSchema(new TagColumnSchema("tag1", TSDataType.STRING)); + final FieldColumnSchema sourceField = new FieldColumnSchema("s_view", TSDataType.DOUBLE); + TreeViewSchema.setOriginalName(sourceField, "s_src"); + table.addColumnSchema(sourceField); + return table; + } + + private static void createTreeView( + final BaseEnv env, final String database, final String treeDatabase, final String viewName) + throws Exception { + try (final Connection connection = env.getConnection(); + final Statement statement = connection.createStatement()) { + statement.execute("create database root." + treeDatabase); + statement.execute( + "create timeseries root." + treeDatabase + ".d1.s_src with datatype=DOUBLE"); + statement.execute( + "create timeseries root." + treeDatabase + ".d1.s_other with datatype=DOUBLE"); + } + + try (final Connection connection = env.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute("create database " + database); + statement.execute("use " + database); + statement.execute( + String.format( + Locale.ROOT, + "create view %s(tag1 string tag, s_view double field from s_src, " + + "s_other double field from s_other) as root.%s.**", + viewName, + treeDatabase)); + } + } + + private void createTopic( + final String topicName, + final String database, + final String tableName, + final String format, + final String columnFilter) + throws Exception { + createTopic( + topicName, database, tableName, TopicConstant.MODE_LIVE_VALUE, format, columnFilter); + } + + private void createTopic( + final String topicName, + final String database, + final String tableName, + final String mode, + final String format, + final String columnFilter) + throws Exception { + createTopic(topicName, database, tableName, mode, format, columnFilter, true); + } + + private void createTopicWithoutColumnFilter( + final String topicName, final String database, final String tableName, final String format) + throws Exception { + createTopic(topicName, database, tableName, TopicConstant.MODE_LIVE_VALUE, format, "", false); + } + + private void createTopic( + final String topicName, + final String database, + final String tableName, + final String mode, + final String format, + final String columnFilter, + final boolean includeColumnFilter) + throws Exception { + try (final ISubscriptionTableSession session = + new SubscriptionTableSessionBuilder() + .host(senderEnv.getIP()) + .port(Integer.parseInt(senderEnv.getPort())) + .build()) { + session.open(); + session.dropTopicIfExists(topicName); + + final Properties config = new Properties(); + config.put(TopicConstant.MODE_KEY, mode); + config.put(TopicConstant.FORMAT_KEY, format); + config.put(TopicConstant.DATABASE_KEY, database); + config.put(TopicConstant.TABLE_KEY, tableName); + if (includeColumnFilter) { + config.put(TopicConstant.COLUMN_FILTER_KEY, columnFilter); + } + session.createTopic(topicName, config); + } + } + + private void alterTopicColumnFilter(final String topicName, final String columnFilter) + throws Exception { + try (final ISubscriptionTableSession session = + new SubscriptionTableSessionBuilder() + .host(senderEnv.getIP()) + .port(Integer.parseInt(senderEnv.getPort())) + .build()) { + session.open(); + + final Properties config = new Properties(); + config.put(TopicConstant.COLUMN_FILTER_KEY, columnFilter); + session.alterTopic(topicName, config); + } + } + + private ISubscriptionTablePullConsumer createConsumer( + final String consumerId, final String consumerGroupId) throws Exception { + final ISubscriptionTablePullConsumer consumer = + new SubscriptionTablePullConsumerBuilder() + .host(senderEnv.getIP()) + .port(Integer.parseInt(senderEnv.getPort())) + .consumerId(consumerId) + .consumerGroupId(consumerGroupId) + .autoCommit(false) + .build(); + consumer.open(); + return consumer; + } + + private static void insertRows( + final BaseEnv env, + final String database, + final String tableName, + final int startInclusive, + final int endExclusive) + throws Exception { + try (final Connection connection = env.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute("use " + database); + for (int i = startInclusive; i < endExclusive; i++) { + statement.execute( + String.format( + Locale.ROOT, + "insert into %s(tag1, s1, s2, s3, time) values ('tag_%d', %d, %.1f, %s, %d)", + tableName, + i, + i * 10L, + i + 0.5d, + i % 2 == 0 ? "true" : "false", + i)); + } + statement.execute("flush"); + } + } + + private static void insertRowsWithS4( + final BaseEnv env, + final String database, + final String tableName, + final int startInclusive, + final int endExclusive) + throws Exception { + try (final Connection connection = env.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute("use " + database); + for (int i = startInclusive; i < endExclusive; i++) { + statement.execute( + String.format( + Locale.ROOT, + "insert into %s(tag1, s1, s2, s3, s4, time) " + + "values ('tag_%d', %d, %.1f, %s, %d, %d)", + tableName, + i, + i * 10L, + i + 0.5d, + i % 2 == 0 ? "true" : "false", + i, + i)); + } + statement.execute("flush"); + } + } + + private static void insertRowsWithAttribute( + final BaseEnv env, + final String database, + final String tableName, + final int startInclusive, + final int endExclusive) + throws Exception { + try (final Connection connection = env.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute("use " + database); + for (int i = startInclusive; i < endExclusive; i++) { + statement.execute( + String.format( + Locale.ROOT, + "insert into %s(tag1, attr1, s1, s2, time) " + + "values ('tag_%d', 'attr_%d', %d, %.1f, %d)", + tableName, + i, + i, + i * 10L, + i + 0.5d, + i)); + } + statement.execute("flush"); + } + } + + private static void insertRowsWithCustomTimeColumn( + final BaseEnv env, + final String database, + final String tableName, + final int startInclusive, + final int endExclusive) + throws Exception { + try (final Connection connection = env.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute("use " + database); + for (int i = startInclusive; i < endExclusive; i++) { + statement.execute( + String.format( + Locale.ROOT, + "insert into %s(tag1, s1, s2, event_time) values ('tag_%d', %d, %.1f, %d)", + tableName, + i, + i * 10L, + i + 0.5d, + i)); + } + statement.execute("flush"); + } + } + + private static void addColumn( + final BaseEnv env, final String database, final String tableName, final String column) + throws Exception { + try (final Connection connection = env.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute("use " + database); + statement.execute(String.format("alter table %s add column %s", tableName, column)); + } + } + + private static void alterColumnType( + final BaseEnv env, + final String database, + final String tableName, + final String columnName, + final String dataType) + throws Exception { + try (final Connection connection = env.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute("use " + database); + statement.execute( + String.format( + "alter table %s alter column %s set data type %s", tableName, columnName, dataType)); + } + } + + private static void assertNoUserRows( + final ISubscriptionTablePullConsumer consumer, final int rounds) throws Exception { + int rowCount = 0; + for (int round = 0; round < rounds; round++) { + final List messages = + consumer.poll(IoTDBSubscriptionITConstant.POLL_TIMEOUT_MS); + if (messages.isEmpty()) { + continue; + } + for (final SubscriptionMessage message : messages) { + if (SubscriptionMessageType.WATERMARK.getType() == message.getMessageType()) { + continue; + } + Assert.assertEquals( + SubscriptionMessageType.RECORD_HANDLER.getType(), message.getMessageType()); + Assert.assertFalse(message.isTimeSelected()); + for (final ResultSet resultSet : message.getResultSets()) { + final SubscriptionRecordHandler.SubscriptionResultSet subscriptionResultSet = + (SubscriptionRecordHandler.SubscriptionResultSet) resultSet; + while (subscriptionResultSet.hasNext()) { + subscriptionResultSet.nextRecord(); + rowCount++; + } + } + } + consumer.commitSync(messages); + } + Assert.assertEquals(0, rowCount); + } + + private static ConsumedRecordStats pollRecordMessages( + final ISubscriptionTablePullConsumer consumer, final int expectedRows) throws Exception { + return pollRecordMessages(consumer, expectedRows, true); + } + + private static ConsumedRecordStats pollRecordMessages( + final ISubscriptionTablePullConsumer consumer, + final int expectedRows, + final boolean expectedTimeSelected) + throws Exception { + final ConsumedRecordStats stats = new ConsumedRecordStats(); + for (int round = 0; round < 60 && stats.rowCount < expectedRows; round++) { + final List messages = + consumer.poll(IoTDBSubscriptionITConstant.POLL_TIMEOUT_MS); + if (messages.isEmpty()) { + continue; + } + consumeRecordMessages(messages, stats, expectedTimeSelected); + consumer.commitSync(messages); + } + Assert.assertEquals(stats.toString(), expectedRows, stats.rowCount); + return stats; + } + + private static ConsumedRecordStats pollRecordMessagesForTimestamps( + final ISubscriptionTablePullConsumer consumer, + final Set expectedTimestamps, + final boolean expectedTimeSelected) + throws Exception { + final ConsumedRecordStats stats = new ConsumedRecordStats(); + int emptyRoundsAfterExpected = 0; + for (int round = 0; round < 90 && emptyRoundsAfterExpected < 1; round++) { + final List messages = + consumer.poll(IoTDBSubscriptionITConstant.POLL_TIMEOUT_MS); + if (messages.isEmpty()) { + if (stats.timestamps.containsAll(expectedTimestamps)) { + emptyRoundsAfterExpected++; + } + continue; + } + consumeRecordMessages(messages, stats, expectedTimeSelected); + consumer.commitSync(messages); + } + Assert.assertTrue(stats.toString(), stats.timestamps.containsAll(expectedTimestamps)); + + final Set unexpectedTimestamps = new LinkedHashSet<>(stats.timestamps); + unexpectedTimestamps.removeAll(expectedTimestamps); + Assert.assertTrue(stats.toString(), unexpectedTimestamps.isEmpty()); + return stats; + } + + private static ConsumedRecordStats pollRecordMessagesForTimestampTimeSelection( + final ISubscriptionTablePullConsumer consumer, final Map expectedTimeSelection) + throws Exception { + final ConsumedRecordStats stats = new ConsumedRecordStats(); + int emptyRoundsAfterExpected = 0; + for (int round = 0; round < 90 && emptyRoundsAfterExpected < 1; round++) { + final List messages = + consumer.poll(IoTDBSubscriptionITConstant.POLL_TIMEOUT_MS); + if (messages.isEmpty()) { + if (stats.timestamps.containsAll(expectedTimeSelection.keySet())) { + emptyRoundsAfterExpected++; + } + continue; + } + consumeRecordMessages(messages, stats, null); + consumer.commitSync(messages); + } + Assert.assertTrue( + stats.toString(), stats.timestamps.containsAll(expectedTimeSelection.keySet())); + + final Set unexpectedTimestamps = new LinkedHashSet<>(stats.timestamps); + unexpectedTimestamps.removeAll(expectedTimeSelection.keySet()); + Assert.assertTrue(stats.toString(), unexpectedTimestamps.isEmpty()); + return stats; + } + + private static void consumeRecordMessages( + final List messages, + final ConsumedRecordStats stats, + final Boolean expectedTimeSelected) + throws Exception { + for (final SubscriptionMessage message : messages) { + if (SubscriptionMessageType.WATERMARK.getType() == message.getMessageType()) { + continue; + } + Assert.assertEquals( + SubscriptionMessageType.RECORD_HANDLER.getType(), message.getMessageType()); + if (Objects.nonNull(expectedTimeSelected)) { + Assert.assertEquals(expectedTimeSelected.booleanValue(), message.isTimeSelected()); + } + for (final ResultSet resultSet : message.getResultSets()) { + final SubscriptionRecordHandler.SubscriptionResultSet subscriptionResultSet = + (SubscriptionRecordHandler.SubscriptionResultSet) resultSet; + stats.tableNames.add(subscriptionResultSet.getTableName()); + subscriptionResultSet + .getColumnNames() + .forEach(columnName -> stats.columnNames.add(columnName.toLowerCase(Locale.ROOT))); + while (subscriptionResultSet.hasNext()) { + final RowRecord rowRecord = subscriptionResultSet.nextRecord(); + stats.timestamps.add(rowRecord.getTimestamp()); + stats.timeSelectedByTimestamp.put( + rowRecord.getTimestamp(), subscriptionResultSet.isTimeSelected()); + stats.rowCount++; + } + } + } + } + + private void pollTsFileMessagesAndLoad( + final ISubscriptionTablePullConsumer consumer, final String database, final int expectedRows) + throws Exception { + for (int round = 0; round < 60 && countRows(receiverEnv, database) < expectedRows; round++) { + final List messages = + consumer.poll(IoTDBSubscriptionITConstant.POLL_TIMEOUT_MS); + if (messages.isEmpty()) { + continue; + } + for (final SubscriptionMessage message : messages) { + if (SubscriptionMessageType.WATERMARK.getType() == message.getMessageType()) { + continue; + } + Assert.assertEquals(SubscriptionMessageType.TS_FILE.getType(), message.getMessageType()); + Assert.assertTrue(message.isTimeSelected()); + final SubscriptionTsFileHandler tsFileHandler = message.getTsFile(); + Assert.assertEquals(database, Objects.requireNonNull(tsFileHandler.getDatabaseName())); + loadTsFile(receiverEnv, database, tsFileHandler); + } + consumer.commitSync(messages); + } + Assert.assertEquals(expectedRows, countRows(receiverEnv, database)); + } + + private static void loadTsFile( + final BaseEnv env, final String database, final SubscriptionTsFileHandler tsFileHandler) + throws Exception { + try (final Connection connection = env.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute("use " + database); + statement.execute(String.format("load '%s'", tsFileHandler.getFile().getAbsolutePath())); + } + } + + private static int countRows(final BaseEnv env, final String database) throws Exception { + try (final Connection connection = env.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute("use " + database); + try (final java.sql.ResultSet resultSet = + statement.executeQuery("select count(*) from " + TABLE_NAME)) { + Assert.assertTrue(resultSet.next()); + return resultSet.getInt(1); + } + } + } + + private void assertLoadedTsFileRows(final String database, final int expectedRows) + throws Exception { + try (final Connection connection = receiverEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute("use " + database); + try (final java.sql.ResultSet resultSet = + statement.executeQuery( + "select time, tag1, s1, s2, s3 from " + TABLE_NAME + " order by time")) { + int rows = 0; + while (resultSet.next()) { + rows++; + final long timestamp = resultSet.getLong("time"); + Assert.assertEquals("tag_" + timestamp, resultSet.getString("tag1")); + Assert.assertNull(resultSet.getObject("s1")); + Assert.assertEquals(timestamp + 0.5d, resultSet.getDouble("s2"), 0.001d); + Assert.assertNull(resultSet.getObject("s3")); + } + Assert.assertEquals(expectedRows, rows); + } + } + } + + private void assertLoadedTsFileRowsWithAllColumns(final String database, final int expectedRows) + throws Exception { + try (final Connection connection = receiverEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute("use " + database); + try (final java.sql.ResultSet resultSet = + statement.executeQuery( + "select time, tag1, s1, s2, s3 from " + TABLE_NAME + " order by time")) { + int rows = 0; + while (resultSet.next()) { + rows++; + final long timestamp = resultSet.getLong("time"); + Assert.assertEquals("tag_" + timestamp, resultSet.getString("tag1")); + Assert.assertEquals(timestamp * 10L, resultSet.getLong("s1")); + Assert.assertEquals(timestamp + 0.5d, resultSet.getDouble("s2"), 0.001d); + Assert.assertEquals(timestamp % 2 == 0, resultSet.getBoolean("s3")); + } + Assert.assertEquals(expectedRows, rows); + } + } + } + + private void cleanup(final String topicName, final String database) { + try (final ISubscriptionTableSession session = + new SubscriptionTableSessionBuilder() + .host(senderEnv.getIP()) + .port(Integer.parseInt(senderEnv.getPort())) + .build()) { + session.open(); + session.dropTopicIfExists(topicName); + } catch (final Exception ignored) { + // ignored on cleanup + } + dropDatabase(senderEnv, database); + dropDatabase(receiverEnv, database); + } + + private static void dropDatabase(final BaseEnv env, final String database) { + try (final Connection connection = env.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute("drop database if exists " + database); + } catch (final Exception ignored) { + // ignored on cleanup + } + } + + private static void dropTreeDatabase(final BaseEnv env, final String database) { + try (final Connection connection = env.getConnection(); + final Statement statement = connection.createStatement()) { + statement.execute("delete database root." + database); + } catch (final Exception ignored) { + // ignored on cleanup + } + } + + private static final class ConsumedRecordStats { + private final Set columnNames = new LinkedHashSet<>(); + private final Set tableNames = new LinkedHashSet<>(); + private final Set timestamps = new LinkedHashSet<>(); + private final Map timeSelectedByTimestamp = new LinkedHashMap<>(); + private int rowCount; + + @Override + public String toString() { + return "ConsumedRecordStats{" + + "columnNames=" + + columnNames + + ", tableNames=" + + tableNames + + ", timestamps=" + + timestamps + + ", timeSelectedByTimestamp=" + + timeSelectedByTimestamp + + ", rowCount=" + + rowCount + + '}'; + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/tablemodel/IoTDBSubscriptionPermissionIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/tablemodel/IoTDBSubscriptionPermissionIT.java index 43c3ea7aebf76..123a68c8df53e 100644 --- a/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/tablemodel/IoTDBSubscriptionPermissionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/tablemodel/IoTDBSubscriptionPermissionIT.java @@ -20,9 +20,11 @@ package org.apache.iotdb.subscription.it.local.tablemodel; import org.apache.iotdb.db.it.utils.TestUtils; +import org.apache.iotdb.isession.ITableSession; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.LocalStandaloneIT; +import org.apache.iotdb.rpc.subscription.config.TopicConstant; import org.apache.iotdb.session.subscription.ISubscriptionTableSession; import org.apache.iotdb.session.subscription.SubscriptionTableSessionBuilder; import org.apache.iotdb.session.subscription.consumer.AckStrategy; @@ -45,6 +47,7 @@ import java.util.Arrays; import java.util.Optional; +import java.util.Properties; import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; @@ -333,6 +336,79 @@ private static void countRowsFromMessage( } } + @Test + public void testColumnFilterTopicRequiresSystemPermission() throws Exception { + final String host = EnvFactory.getEnv().getIP(); + final int port = Integer.parseInt(EnvFactory.getEnv().getPort()); + final String username = "cf_user"; + final String password = "passwd123456"; + final String database = "cf_permission_db"; + final String tableName = "cf_permission_table"; + final String createTopicName = "topic_cf_permission_create"; + final String alterTopicName = "topic_cf_permission_alter"; + + createUser(EnvFactory.getEnv(), username, password); + try (final ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + session.executeNonQueryStatement("create database " + database); + session.executeNonQueryStatement("use " + database); + session.executeNonQueryStatement( + "create table " + tableName + " (tag1 STRING TAG, s1 INT64 FIELD)"); + } + + try (final ISubscriptionTableSession rootSession = + new SubscriptionTableSessionBuilder().host(host).port(port).build(); + final ISubscriptionTableSession userSession = + new SubscriptionTableSessionBuilder() + .host(host) + .port(port) + .username(username) + .password(password) + .build()) { + rootSession.open(); + userSession.open(); + + final Properties topicConfig = + columnFilterTopicConfig(database, tableName, "column_name = \"s1\""); + assertSystemPermissionDenied(() -> userSession.createTopic(createTopicName, topicConfig)); + + rootSession.createTopic(alterTopicName, topicConfig); + final Properties alterConfig = new Properties(); + alterConfig.put(TopicConstant.COLUMN_FILTER_KEY, "column_name = \"tag1\""); + assertSystemPermissionDenied(() -> userSession.alterTopic(alterTopicName, alterConfig)); + } finally { + try (final ISubscriptionTableSession session = + new SubscriptionTableSessionBuilder().host(host).port(port).build()) { + session.open(); + session.dropTopicIfExists(createTopicName); + session.dropTopicIfExists(alterTopicName); + } catch (final Exception ignored) { + // ignored on cleanup + } + } + } + + private static Properties columnFilterTopicConfig( + final String database, final String tableName, final String columnFilter) { + final Properties topicConfig = new Properties(); + topicConfig.put(TopicConstant.MODE_KEY, TopicConstant.MODE_LIVE_VALUE); + topicConfig.put(TopicConstant.FORMAT_KEY, TopicConstant.FORMAT_RECORD_HANDLER_VALUE); + topicConfig.put(TopicConstant.DATABASE_KEY, database); + topicConfig.put(TopicConstant.TABLE_KEY, tableName); + topicConfig.put(TopicConstant.COLUMN_FILTER_KEY, columnFilter); + return topicConfig; + } + + private static void assertSystemPermissionDenied(final CheckedRunnable runnable) { + final Exception exception = Assert.assertThrows(Exception.class, runnable::run); + Assert.assertTrue(String.valueOf(exception.getMessage()).contains("Access Denied")); + Assert.assertTrue(String.valueOf(exception.getMessage()).contains("SYSTEM")); + } + + @FunctionalInterface + private interface CheckedRunnable { + void run() throws Exception; + } + @Ignore @Test public void testTablePermission() { diff --git a/iotdb-client/subscription/src/main/java/org/apache/iotdb/rpc/subscription/config/TopicConfig.java b/iotdb-client/subscription/src/main/java/org/apache/iotdb/rpc/subscription/config/TopicConfig.java index fefd8778bb602..2e5aaa8f2c7f8 100644 --- a/iotdb-client/subscription/src/main/java/org/apache/iotdb/rpc/subscription/config/TopicConfig.java +++ b/iotdb-client/subscription/src/main/java/org/apache/iotdb/rpc/subscription/config/TopicConfig.java @@ -30,6 +30,7 @@ import java.util.HashMap; import java.util.HashSet; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; @@ -185,6 +186,23 @@ public Map getAttributesWithSourceDatabaseAndTableName() { return attributes; } + public Map getAttributesWithSourceColumnFilter() { + return Collections.singletonMap(TopicConstant.COLUMN_FILTER_KEY, getColumnFilter()); + } + + public String getColumnFilter() { + return getStringIgnoreCase( + TopicConstant.COLUMN_FILTER_KEY, TopicConstant.COLUMN_FILTER_DEFAULT_VALUE); + } + + public boolean hasColumnFilter() { + return containsKeyIgnoreCase(TopicConstant.COLUMN_FILTER_KEY); + } + + public boolean isColumnFilterTrivial() { + return TopicConstant.COLUMN_FILTER_DEFAULT_VALUE.equalsIgnoreCase(getColumnFilter().trim()); + } + public Map getAttributesWithSourceTimeRange() { final Map attributesWithTimeRange = new HashMap<>(); @@ -291,4 +309,17 @@ public Map getAttributesWithSinkPrefix() { }); return attributesWithProcessorPrefix; } + + private boolean containsKeyIgnoreCase(final String expectedKey) { + return attributes.keySet().stream().anyMatch(key -> expectedKey.equalsIgnoreCase(key)); + } + + private String getStringIgnoreCase(final String expectedKey, final String defaultValue) { + return attributes.entrySet().stream() + .filter(entry -> expectedKey.equalsIgnoreCase(entry.getKey())) + .map(Map.Entry::getValue) + .filter(Objects::nonNull) + .findFirst() + .orElse(defaultValue); + } } diff --git a/iotdb-client/subscription/src/main/java/org/apache/iotdb/rpc/subscription/config/TopicConstant.java b/iotdb-client/subscription/src/main/java/org/apache/iotdb/rpc/subscription/config/TopicConstant.java index a0b391225c293..f929d7b74722e 100644 --- a/iotdb-client/subscription/src/main/java/org/apache/iotdb/rpc/subscription/config/TopicConstant.java +++ b/iotdb-client/subscription/src/main/java/org/apache/iotdb/rpc/subscription/config/TopicConstant.java @@ -30,12 +30,12 @@ public class TopicConstant { public static final String DATABASE_KEY = "database"; public static final String TABLE_KEY = "table"; - public static final String COLUMN_KEY = "column"; + public static final String COLUMN_FILTER_KEY = "column-filter"; public static final String RETENTION_BYTES_KEY = "retention.bytes"; public static final String RETENTION_MS_KEY = "retention.ms"; public static final String DATABASE_DEFAULT_VALUE = ".*"; public static final String TABLE_DEFAULT_VALUE = ".*"; - public static final String COLUMN_DEFAULT_VALUE = ".*"; + public static final String COLUMN_FILTER_DEFAULT_VALUE = "true"; public static final String START_TIME_KEY = "start-time"; public static final String END_TIME_KEY = "end-time"; diff --git a/iotdb-client/subscription/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/SubscriptionPollResponse.java b/iotdb-client/subscription/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/SubscriptionPollResponse.java index 194f87bf71d4f..1c484925f6d03 100644 --- a/iotdb-client/subscription/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/SubscriptionPollResponse.java +++ b/iotdb-client/subscription/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/SubscriptionPollResponse.java @@ -29,8 +29,10 @@ import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; +import java.util.Collections; import java.util.HashMap; import java.util.Map; +import java.util.Objects; public class SubscriptionPollResponse { @@ -42,13 +44,36 @@ public class SubscriptionPollResponse { private final transient SubscriptionCommitContext commitContext; + private final transient boolean timeSelected; + + private final transient Map> timeSelectedByTable; + public SubscriptionPollResponse( final short responseType, final SubscriptionPollPayload payload, final SubscriptionCommitContext commitContext) { + this(responseType, payload, commitContext, true); + } + + public SubscriptionPollResponse( + final short responseType, + final SubscriptionPollPayload payload, + final SubscriptionCommitContext commitContext, + final boolean timeSelected) { + this(responseType, payload, commitContext, timeSelected, Collections.emptyMap()); + } + + public SubscriptionPollResponse( + final short responseType, + final SubscriptionPollPayload payload, + final SubscriptionCommitContext commitContext, + final boolean timeSelected, + final Map> timeSelectedByTable) { this.responseType = responseType; this.payload = payload; this.commitContext = commitContext; + this.timeSelected = timeSelected; + this.timeSelectedByTable = copyTimeSelectedByTable(timeSelectedByTable); } public short getResponseType() { @@ -63,6 +88,14 @@ public SubscriptionCommitContext getCommitContext() { return commitContext; } + public boolean isTimeSelected() { + return timeSelected; + } + + public Map> getTimeSelectedByTable() { + return timeSelectedByTable; + } + /////////////////////////////// de/ser /////////////////////////////// public static ByteBuffer serialize(final SubscriptionPollResponse response) throws IOException { @@ -77,6 +110,8 @@ private void serialize(final DataOutputStream stream) throws IOException { ReadWriteIOUtils.write(responseType, stream); payload.serialize(stream); commitContext.serialize(stream); + ReadWriteIOUtils.write(timeSelected, stream); + serializeTimeSelectedByTable(stream); } public static SubscriptionPollResponse deserialize(final ByteBuffer buffer) { @@ -114,7 +149,71 @@ public static SubscriptionPollResponse deserialize(final ByteBuffer buffer) { } final SubscriptionCommitContext commitContext = SubscriptionCommitContext.deserialize(buffer); - return new SubscriptionPollResponse(responseType, payload, commitContext); + final boolean timeSelected = buffer.hasRemaining() ? ReadWriteIOUtils.readBool(buffer) : true; + final Map> timeSelectedByTable = + buffer.hasRemaining() ? deserializeTimeSelectedByTable(buffer) : Collections.emptyMap(); + return new SubscriptionPollResponse( + responseType, payload, commitContext, timeSelected, timeSelectedByTable); + } + + private void serializeTimeSelectedByTable(final DataOutputStream stream) throws IOException { + ReadWriteIOUtils.write(timeSelectedByTable.size(), stream); + for (final Map.Entry> databaseEntry : + timeSelectedByTable.entrySet()) { + ReadWriteIOUtils.write(databaseEntry.getKey(), stream); + ReadWriteIOUtils.write(databaseEntry.getValue().size(), stream); + for (final Map.Entry tableEntry : databaseEntry.getValue().entrySet()) { + ReadWriteIOUtils.write(tableEntry.getKey(), stream); + ReadWriteIOUtils.write(tableEntry.getValue(), stream); + } + } + } + + private static Map> deserializeTimeSelectedByTable( + final ByteBuffer buffer) { + final int databaseSize = ReadWriteIOUtils.readInt(buffer); + if (databaseSize <= 0) { + return Collections.emptyMap(); + } + final Map> result = new HashMap<>(); + for (int i = 0; i < databaseSize; ++i) { + final String databaseName = ReadWriteIOUtils.readString(buffer); + final int tableSize = ReadWriteIOUtils.readInt(buffer); + final Map tableMap = new HashMap<>(); + for (int j = 0; j < tableSize; ++j) { + tableMap.put(ReadWriteIOUtils.readString(buffer), ReadWriteIOUtils.readBool(buffer)); + } + result.put(databaseName, tableMap); + } + return copyTimeSelectedByTable(result); + } + + private static Map> copyTimeSelectedByTable( + final Map> timeSelectedByTable) { + if (Objects.isNull(timeSelectedByTable) || timeSelectedByTable.isEmpty()) { + return Collections.emptyMap(); + } + final Map> copied = new HashMap<>(); + timeSelectedByTable.forEach( + (databaseName, tableMap) -> { + if (Objects.isNull(databaseName) || Objects.isNull(tableMap) || tableMap.isEmpty()) { + return; + } + final Map copiedTableMap = new HashMap<>(); + tableMap.forEach( + (tableName, timeSelected) -> { + if (Objects.nonNull(tableName) && Objects.nonNull(timeSelected)) { + copiedTableMap.put(tableName, timeSelected); + } + }); + if (!copiedTableMap.isEmpty()) { + copied.put(databaseName, Collections.unmodifiableMap(copiedTableMap)); + } + }); + if (copied.isEmpty()) { + return Collections.emptyMap(); + } + return Collections.unmodifiableMap(copied); } /////////////////////////////// stringify /////////////////////////////// @@ -130,6 +229,8 @@ protected Map coreReportMessage() { result.put("responseType", type != null ? type.toString() : "UNKNOWN(" + responseType + ")"); result.put("payload", payload != null ? payload.toString() : "null"); result.put("commitContext", commitContext != null ? commitContext.toString() : "null"); + result.put("timeSelected", String.valueOf(timeSelected)); + result.put("timeSelectedByTable", String.valueOf(timeSelectedByTable)); return result; } } diff --git a/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionConsumer.java b/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionConsumer.java index 9ff52e4386fc2..9f8ab0ac6f49c 100644 --- a/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionConsumer.java +++ b/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionConsumer.java @@ -1092,7 +1092,8 @@ private Optional pollFileInternal( new SubscriptionMessage( commitContext, file.getAbsolutePath(), - ((FileSealPayload) payload).getDatabaseName())); + ((FileSealPayload) payload).getDatabaseName(), + response.isTimeSelected())); } case ERROR: { @@ -1152,6 +1153,9 @@ private Optional pollTabletsInternal( final Map> tablets = ((TabletsPayload) initialResponse.getPayload()).getTabletsWithDBInfo(); final SubscriptionCommitContext commitContext = initialResponse.getCommitContext(); + boolean timeSelected = initialResponse.isTimeSelected(); + final Map> timeSelectedByTable = new HashMap<>(); + mergeTimeSelectedByTable(timeSelectedByTable, initialResponse.getTimeSelectedByTable()); int nextOffset = ((TabletsPayload) initialResponse.getPayload()).getNextOffset(); while (true) { @@ -1165,7 +1169,8 @@ private Optional pollTabletsInternal( LOGGER.warn(errorMessage); throw new SubscriptionRuntimeNonCriticalException(errorMessage); } - return Optional.of(new SubscriptionMessage(commitContext, tablets)); + return Optional.of( + new SubscriptionMessage(commitContext, tablets, timeSelected, timeSelectedByTable)); } timer.update(); @@ -1221,6 +1226,8 @@ private Optional pollTabletsInternal( } // update offset + timeSelected = timeSelected && response.isTimeSelected(); + mergeTimeSelectedByTable(timeSelectedByTable, response.getTimeSelectedByTable()); nextOffset = ((TabletsPayload) payload).getNextOffset(); break; } @@ -1254,6 +1261,21 @@ private Optional pollTabletsInternal( } } + private static void mergeTimeSelectedByTable( + final Map> target, + final Map> source) { + if (Objects.isNull(source) || source.isEmpty()) { + return; + } + source.forEach( + (databaseName, tableMap) -> { + if (Objects.isNull(tableMap) || tableMap.isEmpty()) { + return; + } + target.computeIfAbsent(databaseName, ignored -> new HashMap<>()).putAll(tableMap); + }); + } + private List pollInternal( final Set topicNames, final long timeoutMs) throws SubscriptionException { providers.acquireReadLock(); diff --git a/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/payload/SubscriptionMessage.java b/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/payload/SubscriptionMessage.java index 523b86587fe44..082681ca3666a 100644 --- a/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/payload/SubscriptionMessage.java +++ b/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/payload/SubscriptionMessage.java @@ -40,6 +40,8 @@ public class SubscriptionMessage implements Comparable { private final SubscriptionMessageHandler handler; + private final boolean timeSelected; + /** Watermark timestamp, valid only when messageType == WATERMARK. */ private final long watermarkTimestamp; @@ -47,9 +49,25 @@ public class SubscriptionMessage implements Comparable { public SubscriptionMessage( final SubscriptionCommitContext commitContext, final Map> tablets) { + this(commitContext, tablets, true); + } + + public SubscriptionMessage( + final SubscriptionCommitContext commitContext, + final Map> tablets, + final boolean timeSelected) { + this(commitContext, tablets, timeSelected, null); + } + + public SubscriptionMessage( + final SubscriptionCommitContext commitContext, + final Map> tablets, + final boolean timeSelected, + final Map> timeSelectedByTable) { this.commitContext = commitContext; this.messageType = SubscriptionMessageType.RECORD_HANDLER.getType(); - this.handler = new SubscriptionRecordHandler(tablets); + this.handler = new SubscriptionRecordHandler(tablets, timeSelected, timeSelectedByTable); + this.timeSelected = timeSelected; this.watermarkTimestamp = Long.MIN_VALUE; } @@ -57,9 +75,18 @@ public SubscriptionMessage( final SubscriptionCommitContext commitContext, final String absolutePath, @Nullable final String databaseName) { + this(commitContext, absolutePath, databaseName, true); + } + + public SubscriptionMessage( + final SubscriptionCommitContext commitContext, + final String absolutePath, + @Nullable final String databaseName, + final boolean timeSelected) { this.commitContext = commitContext; this.messageType = SubscriptionMessageType.TS_FILE.getType(); this.handler = new SubscriptionTsFileHandler(absolutePath, databaseName); + this.timeSelected = timeSelected; this.watermarkTimestamp = Long.MIN_VALUE; } @@ -69,6 +96,7 @@ public SubscriptionMessage( this.commitContext = commitContext; this.messageType = SubscriptionMessageType.WATERMARK.getType(); this.handler = null; + this.timeSelected = true; this.watermarkTimestamp = watermarkTimestamp; } @@ -80,6 +108,10 @@ public short getMessageType() { return messageType; } + public boolean isTimeSelected() { + return timeSelected; + } + /** * Returns the watermark timestamp carried by this message. Only valid when {@code * getMessageType() == SubscriptionMessageType.WATERMARK.getType()}. @@ -140,13 +172,14 @@ public boolean equals(final Object obj) { final SubscriptionMessage that = (SubscriptionMessage) obj; return Objects.equals(this.commitContext, that.commitContext) && this.watermarkTimestamp == that.watermarkTimestamp + && this.timeSelected == that.timeSelected && Objects.equals(this.messageType, that.messageType) && Objects.equals(this.handler, that.handler); } @Override public int hashCode() { - return Objects.hash(commitContext, messageType, handler, watermarkTimestamp); + return Objects.hash(commitContext, messageType, handler, timeSelected, watermarkTimestamp); } @Override @@ -160,6 +193,8 @@ public String toString() { + commitContext + ", messageType=" + SubscriptionMessageType.valueOf(messageType).toString() + + ", timeSelected=" + + timeSelected + ", watermarkTimestamp=" + watermarkTimestamp + "}"; diff --git a/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/payload/SubscriptionRecordHandler.java b/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/payload/SubscriptionRecordHandler.java index 69bd24d768711..af5cbb077ebde 100644 --- a/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/payload/SubscriptionRecordHandler.java +++ b/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/payload/SubscriptionRecordHandler.java @@ -28,6 +28,7 @@ import org.apache.tsfile.read.common.RowRecord; import org.apache.tsfile.read.query.dataset.AbstractResultSet; import org.apache.tsfile.read.query.dataset.ResultSet; +import org.apache.tsfile.read.query.dataset.ResultSetMetadata; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.BitMap; import org.apache.tsfile.utils.DateUtils; @@ -55,6 +56,18 @@ public class SubscriptionRecordHandler implements Iterable, Subscript private final List resultSetView; public SubscriptionRecordHandler(final Map> tablets) { + this(tablets, true); + } + + public SubscriptionRecordHandler( + final Map> tablets, final boolean timeSelected) { + this(tablets, timeSelected, Collections.emptyMap()); + } + + public SubscriptionRecordHandler( + final Map> tablets, + final boolean timeSelected, + final Map> timeSelectedByTable) { final List resultSets = new ArrayList<>(); for (final Map.Entry> entry : tablets.entrySet()) { final String databaseName = entry.getKey(); @@ -66,7 +79,11 @@ public SubscriptionRecordHandler(final Map> tablets) { if (Objects.isNull(tablet)) { continue; } - resultSets.add(new SubscriptionResultSet(tablet, databaseName)); + resultSets.add( + new SubscriptionResultSet( + tablet, + databaseName, + resolveTimeSelected(timeSelectedByTable, timeSelected, databaseName, tablet))); } } this.resultSets = Collections.unmodifiableList(resultSets); @@ -89,12 +106,57 @@ public void removeUserData() { resultSets.forEach(SubscriptionResultSet::removeUserData); } + private static boolean resolveTimeSelected( + final Map> timeSelectedByTable, + final boolean defaultTimeSelected, + final String databaseName, + final Tablet tablet) { + if (Objects.isNull(timeSelectedByTable) || timeSelectedByTable.isEmpty()) { + return defaultTimeSelected; + } + final Map tableMap = timeSelectedByTable.get(databaseName); + if (Objects.isNull(tablet)) { + return defaultTimeSelected; + } + final Map resolvedTableMap = + Objects.nonNull(tableMap) + ? tableMap + : getIgnoreCase(timeSelectedByTable, databaseName, null); + if (Objects.isNull(resolvedTableMap)) { + return defaultTimeSelected; + } + final Boolean tableTimeSelected = + resolvedTableMap.containsKey(tablet.getTableName()) + ? resolvedTableMap.get(tablet.getTableName()) + : getIgnoreCase(resolvedTableMap, tablet.getTableName(), null); + return Objects.nonNull(tableTimeSelected) + ? Boolean.TRUE.equals(tableTimeSelected) + : defaultTimeSelected; + } + + private static T getIgnoreCase( + final Map map, final String key, final T defaultValue) { + if (Objects.isNull(map) || Objects.isNull(key)) { + return defaultValue; + } + for (final Map.Entry entry : map.entrySet()) { + if (Objects.nonNull(entry.getKey()) && entry.getKey().equalsIgnoreCase(key)) { + return entry.getValue(); + } + } + return defaultValue; + } + public static class SubscriptionResultSet extends AbstractResultSet { private Tablet tablet; @Nullable private final String databaseName; + private final boolean timeSelected; + + private final int visibleColumnCount; + private final List sortedRowPositions; private int rowIndex = -1; @@ -103,10 +165,21 @@ public static class SubscriptionResultSet extends AbstractResultSet { private volatile boolean userDataRemoved = false; - private SubscriptionResultSet(final Tablet tablet, @Nullable final String databaseName) { + private SubscriptionResultSet( + final Tablet tablet, @Nullable final String databaseName, final boolean timeSelected) { super(generateColumnNames(tablet, databaseName), generateColumnTypes(tablet)); this.tablet = tablet; this.databaseName = databaseName; + this.timeSelected = timeSelected; + this.visibleColumnCount = tablet.getSchemas().size() + (shouldExposeTime() ? 1 : 0); + if (!shouldExposeTime()) { + resultSetMetadata = new SubscriptionResultSetMetadata(tablet); + columnNameToColumnIndexMap.clear(); + final List schemas = tablet.getSchemas(); + for (int i = 0; i < schemas.size(); ++i) { + columnNameToColumnIndexMap.put(schemas.get(i).getMeasurementName(), i + 1); + } + } this.sortedRowPositions = generateSortedRowPositions(tablet); } @@ -134,25 +207,16 @@ public List getColumnCategories() { return columnCategoryList = Stream.concat( - Stream.of(ColumnCategory.TIME), + shouldExposeTime() ? Stream.of(ColumnCategory.TIME) : Stream.empty(), tablet.getColumnTypes().stream() - .map( - columnCategory -> { - switch (columnCategory) { - case FIELD: - return ColumnCategory.FIELD; - case TAG: - return ColumnCategory.TAG; - case ATTRIBUTE: - return ColumnCategory.ATTRIBUTE; - default: - throw new IllegalArgumentException( - "Unknown column category: " + columnCategory); - } - })) + .map(SubscriptionResultSet::convertColumnCategory)) .collect(Collectors.toList()); } + public boolean isTimeSelected() { + return timeSelected; + } + public Tablet getTablet() { ensureUserDataAvailable(); return tablet; @@ -170,7 +234,7 @@ public RowRecord nextRecord() throws IOException { public int getColumnCount() { ensureUserDataAvailable(); - return tablet.getSchemas().size() + 1; + return visibleColumnCount; } public List getColumnNames() { @@ -311,9 +375,7 @@ private RowRecord generateRowRecord(final long timestamp, final int rowPosition) final BitMap[] bitMaps = tablet.getBitMaps(); for (int columnIndex = 0; columnIndex < columnSize; ++columnIndex) { final Field field; - if (bitMaps != null - && bitMaps[columnIndex] != null - && bitMaps[columnIndex].isMarked(rowPosition)) { + if (isNullValue(tablet.getValues(), bitMaps, columnIndex, rowPosition)) { field = new Field(null); } else { final TSDataType dataType = tablet.getSchemas().get(columnIndex).getType(); @@ -334,9 +396,7 @@ private TSRecord generateTsRecord( final BitMap[] bitMaps = currentTablet.getBitMaps(); for (int columnIndex = 0; columnIndex < currentTablet.getSchemas().size(); ++columnIndex) { - if (bitMaps != null - && bitMaps[columnIndex] != null - && bitMaps[columnIndex].isMarked(currentRowIndex)) { + if (isNullValue(currentTablet.getValues(), bitMaps, columnIndex, currentRowIndex)) { continue; } @@ -380,6 +440,48 @@ private TSRecord generateTsRecord( return record; } + private static boolean isNullValue( + final Object[] values, final BitMap[] bitMaps, final int columnIndex, final int rowIndex) { + if (Objects.isNull(values) + || columnIndex >= values.length + || Objects.isNull(values[columnIndex])) { + return true; + } + return bitMaps != null + && columnIndex < bitMaps.length + && bitMaps[columnIndex] != null + && bitMaps[columnIndex].isMarked(rowIndex); + } + + @Override + protected Field getField(final int index) { + if (shouldExposeTime()) { + return super.getField(index); + } + if (index <= 0 || index > visibleColumnCount) { + throw new IndexOutOfBoundsException("ResultSet column index out of bound: " + index); + } + return currentRow.getField(index - 1); + } + + private boolean shouldExposeTime() { + return !isTableData() || timeSelected; + } + + private static ColumnCategory convertColumnCategory( + final org.apache.tsfile.enums.ColumnCategory columnCategory) { + switch (columnCategory) { + case FIELD: + return ColumnCategory.FIELD; + case TAG: + return ColumnCategory.TAG; + case ATTRIBUTE: + return ColumnCategory.ATTRIBUTE; + default: + throw new IllegalArgumentException("Unknown column category: " + columnCategory); + } + } + private static Field generateFieldFromTabletValue( final TSDataType dataType, final Object value, final int index) { final Field field = new Field(dataType); @@ -425,5 +527,24 @@ private RowPosition(final long timestamp, final int rowIndex) { this.rowIndex = rowIndex; } } + + private static class SubscriptionResultSetMetadata implements ResultSetMetadata { + + private final List schemas; + + private SubscriptionResultSetMetadata(final Tablet tablet) { + this.schemas = tablet.getSchemas(); + } + + @Override + public String getColumnName(final int index) { + return schemas.get(index - 1).getMeasurementName(); + } + + @Override + public TSDataType getColumnType(final int index) { + return schemas.get(index - 1).getType(); + } + } } } diff --git a/iotdb-client/subscription/src/test/java/org/apache/iotdb/rpc/subscription/config/TopicConfigTest.java b/iotdb-client/subscription/src/test/java/org/apache/iotdb/rpc/subscription/config/TopicConfigTest.java new file mode 100644 index 0000000000000..d60f4f1f7021a --- /dev/null +++ b/iotdb-client/subscription/src/test/java/org/apache/iotdb/rpc/subscription/config/TopicConfigTest.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.rpc.subscription.config; + +import org.junit.Assert; +import org.junit.Test; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +public class TopicConfigTest { + + @Test + public void testColumnFilterKeyIsCaseInsensitive() { + final TopicConfig topicConfig = + new TopicConfig(Collections.singletonMap("Column-Filter", "column_name = \"s1\"")); + + Assert.assertTrue(topicConfig.hasColumnFilter()); + Assert.assertEquals("column_name = \"s1\"", topicConfig.getColumnFilter()); + Assert.assertEquals( + "column_name = \"s1\"", + topicConfig.getAttributesWithSourceColumnFilter().get(TopicConstant.COLUMN_FILTER_KEY)); + } + + @Test + public void testColumnFilterDefaultsToTrivialWhenAbsent() { + final TopicConfig topicConfig = new TopicConfig(new HashMap<>()); + + Assert.assertFalse(topicConfig.hasColumnFilter()); + Assert.assertTrue(topicConfig.isColumnFilterTrivial()); + Assert.assertEquals(TopicConstant.COLUMN_FILTER_DEFAULT_VALUE, topicConfig.getColumnFilter()); + } + + @Test + public void testColumnFilterTrivialWithMixedCaseKeyAndValue() { + final Map attributes = new HashMap<>(); + attributes.put("COLUMN-FILTER", " TRUE "); + + Assert.assertTrue(new TopicConfig(attributes).isColumnFilterTrivial()); + } +} diff --git a/iotdb-client/subscription/src/test/java/org/apache/iotdb/rpc/subscription/payload/poll/SubscriptionPollResponseTest.java b/iotdb-client/subscription/src/test/java/org/apache/iotdb/rpc/subscription/payload/poll/SubscriptionPollResponseTest.java new file mode 100644 index 0000000000000..d4aeb436e7a53 --- /dev/null +++ b/iotdb-client/subscription/src/test/java/org/apache/iotdb/rpc/subscription/payload/poll/SubscriptionPollResponseTest.java @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.rpc.subscription.payload.poll; + +import org.apache.tsfile.utils.PublicBAOS; +import org.apache.tsfile.utils.ReadWriteIOUtils; +import org.junit.Test; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class SubscriptionPollResponseTest { + + @Test + public void testRoundTripPreservesTimeSelected() throws IOException { + final Map> timeSelectedByTable = new HashMap<>(); + timeSelectedByTable.put("root.sg", Collections.singletonMap("table1", false)); + final SubscriptionPollResponse response = + new SubscriptionPollResponse( + SubscriptionPollResponseType.TABLETS.getType(), + new TabletsPayload(Collections.emptyMap(), 0), + new SubscriptionCommitContext(1, 2, "topic", "group", 3L), + false, + timeSelectedByTable); + + final SubscriptionPollResponse parsed = + SubscriptionPollResponse.deserialize(SubscriptionPollResponse.serialize(response)); + + assertFalse(parsed.isTimeSelected()); + assertFalse(parsed.getTimeSelectedByTable().get("root.sg").get("table1")); + } + + @Test + public void testDeserializeOldWireFormatDefaultsTimeSelectedToTrue() throws IOException { + final ByteBuffer buffer = + serializeWithoutTimeSelected( + SubscriptionPollResponseType.TABLETS.getType(), + new TabletsPayload(Collections.emptyMap(), 0), + new SubscriptionCommitContext(1, 2, "topic", "group", 3L)); + + final SubscriptionPollResponse parsed = SubscriptionPollResponse.deserialize(buffer); + + assertTrue(parsed.isTimeSelected()); + assertTrue(parsed.getTimeSelectedByTable().isEmpty()); + } + + @Test + public void testEmptyTimeSelectedByTableEntriesAreIgnored() { + final Map> timeSelectedByTable = new HashMap<>(); + timeSelectedByTable.put("root.sg", Collections.emptyMap()); + final SubscriptionPollResponse response = + new SubscriptionPollResponse( + SubscriptionPollResponseType.TABLETS.getType(), + new TabletsPayload(Collections.emptyMap(), 0), + new SubscriptionCommitContext(1, 2, "topic", "group", 3L), + false, + timeSelectedByTable); + + assertTrue(response.getTimeSelectedByTable().isEmpty()); + } + + @Test + public void testInvalidTimeSelectedByTableEntriesAreIgnored() throws IOException { + final Map tableMap = new HashMap<>(); + tableMap.put(null, false); + tableMap.put("table1", null); + tableMap.put("table2", false); + final Map> timeSelectedByTable = new HashMap<>(); + timeSelectedByTable.put(null, Collections.singletonMap("ignored", false)); + timeSelectedByTable.put("root.sg", tableMap); + final SubscriptionPollResponse response = + new SubscriptionPollResponse( + SubscriptionPollResponseType.TABLETS.getType(), + new TabletsPayload(Collections.emptyMap(), 0), + new SubscriptionCommitContext(1, 2, "topic", "group", 3L), + false, + timeSelectedByTable); + + final SubscriptionPollResponse parsed = + SubscriptionPollResponse.deserialize(SubscriptionPollResponse.serialize(response)); + + assertFalse(parsed.getTimeSelectedByTable().get("root.sg").get("table2")); + assertFalse(parsed.getTimeSelectedByTable().get("root.sg").containsKey("table1")); + } + + private static ByteBuffer serializeWithoutTimeSelected( + final short responseType, + final SubscriptionPollPayload payload, + final SubscriptionCommitContext commitContext) + throws IOException { + try (final PublicBAOS byteArrayOutputStream = new PublicBAOS(); + final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) { + ReadWriteIOUtils.write(responseType, outputStream); + payload.serialize(outputStream); + commitContext.serialize(outputStream); + return ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size()); + } + } +} diff --git a/iotdb-client/subscription/src/test/java/org/apache/iotdb/session/subscription/payload/SubscriptionRecordHandlerTest.java b/iotdb-client/subscription/src/test/java/org/apache/iotdb/session/subscription/payload/SubscriptionRecordHandlerTest.java new file mode 100644 index 0000000000000..edacf47936f28 --- /dev/null +++ b/iotdb-client/subscription/src/test/java/org/apache/iotdb/session/subscription/payload/SubscriptionRecordHandlerTest.java @@ -0,0 +1,176 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.session.subscription.payload; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.RowRecord; +import org.apache.tsfile.read.query.dataset.ResultSet; +import org.apache.tsfile.utils.BitMap; +import org.apache.tsfile.write.record.TSRecord; +import org.apache.tsfile.write.record.Tablet; +import org.junit.Test; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +public class SubscriptionRecordHandlerTest { + + @Test + public void testTableResultSetHidesTimeWhenTimeIsNotSelected() throws IOException { + final SubscriptionRecordHandler.SubscriptionResultSet resultSet = + singleResultSet(new SubscriptionRecordHandler(tabletsWithDatabase(tablet()), false)); + + assertFalse(resultSet.isTimeSelected()); + assertEquals(2, resultSet.getColumnCount()); + assertEquals(Arrays.asList("s1", "id"), resultSet.getColumnNames()); + assertEquals(Arrays.asList("INT64", "STRING"), resultSet.getColumnTypes()); + assertEquals( + Arrays.asList( + SubscriptionRecordHandler.SubscriptionResultSet.ColumnCategory.FIELD, + SubscriptionRecordHandler.SubscriptionResultSet.ColumnCategory.TAG), + resultSet.getColumnCategories()); + assertEquals("s1", resultSet.getMetadata().getColumnName(1)); + assertEquals(TSDataType.INT64, resultSet.getMetadata().getColumnType(1)); + + assertTrue(resultSet.next()); + assertEquals(100L, resultSet.getLong(1)); + assertEquals(100L, resultSet.getLong("s1")); + assertEquals("deviceA", resultSet.getString(2)); + assertEquals("deviceA", resultSet.getString("id")); + } + + @Test + public void testTableResultSetExposesTimeWhenTimeIsSelected() throws IOException { + final SubscriptionRecordHandler.SubscriptionResultSet resultSet = + singleResultSet(new SubscriptionRecordHandler(tabletsWithDatabase(tablet()), true)); + + assertTrue(resultSet.isTimeSelected()); + assertEquals(3, resultSet.getColumnCount()); + assertEquals(Arrays.asList("Time", "s1", "id"), resultSet.getColumnNames()); + assertEquals(Arrays.asList("INT64", "INT64", "STRING"), resultSet.getColumnTypes()); + assertEquals( + Arrays.asList( + SubscriptionRecordHandler.SubscriptionResultSet.ColumnCategory.TIME, + SubscriptionRecordHandler.SubscriptionResultSet.ColumnCategory.FIELD, + SubscriptionRecordHandler.SubscriptionResultSet.ColumnCategory.TAG), + resultSet.getColumnCategories()); + + assertTrue(resultSet.next()); + assertEquals(1234L, resultSet.getLong(1)); + assertEquals(100L, resultSet.getLong(2)); + assertEquals("deviceA", resultSet.getString(3)); + } + + @Test + public void testTableResultSetUsesTableSpecificTimeSelection() throws IOException { + final Map> tablets = + Collections.singletonMap("root.sg", Arrays.asList(tablet("table1"), tablet("table2"))); + final Map tableMap = new HashMap<>(); + tableMap.put("table1", false); + tableMap.put("table2", true); + final SubscriptionRecordHandler handler = + new SubscriptionRecordHandler(tablets, true, Collections.singletonMap("root.sg", tableMap)); + + final SubscriptionRecordHandler.SubscriptionResultSet first = + (SubscriptionRecordHandler.SubscriptionResultSet) handler.getResultSets().get(0); + final SubscriptionRecordHandler.SubscriptionResultSet second = + (SubscriptionRecordHandler.SubscriptionResultSet) handler.getResultSets().get(1); + + assertFalse(first.isTimeSelected()); + assertEquals(Arrays.asList("s1", "id"), first.getColumnNames()); + assertTrue(second.isTimeSelected()); + assertEquals(Arrays.asList("Time", "s1", "id"), second.getColumnNames()); + } + + @Test + public void testTableResultSetUsesCaseInsensitiveTableSpecificTimeSelection() throws IOException { + final Map> tablets = + Collections.singletonMap("Root.SG", Collections.singletonList(tablet("Table1"))); + final Map tableMap = new HashMap<>(); + tableMap.put("table1", false); + final SubscriptionRecordHandler.SubscriptionResultSet resultSet = + singleResultSet( + new SubscriptionRecordHandler( + tablets, true, Collections.singletonMap("root.sg", tableMap))); + + assertFalse(resultSet.isTimeSelected()); + assertEquals(Arrays.asList("s1", "id"), resultSet.getColumnNames()); + } + + @Test + public void testTableResultSetHandlesNullColumnArray() throws IOException { + final Tablet tablet = tablet(); + tablet.getValues()[0] = null; + final BitMap[] bitMaps = new BitMap[] {new BitMap(1), null}; + bitMaps[0].mark(0); + tablet.setBitMaps(bitMaps); + final SubscriptionRecordHandler.SubscriptionResultSet resultSet = + singleResultSet(new SubscriptionRecordHandler(tabletsWithDatabase(tablet), true)); + + final RowRecord rowRecord = resultSet.nextRecord(); + assertNull(rowRecord.getFields().get(0).getDataType()); + + final Iterator records = resultSet.iterator(); + assertTrue(records.hasNext()); + final TSRecord record = records.next(); + assertEquals(1, record.dataPointList.size()); + } + + private static SubscriptionRecordHandler.SubscriptionResultSet singleResultSet( + final SubscriptionRecordHandler handler) { + final List resultSets = handler.getResultSets(); + assertEquals(1, resultSets.size()); + return (SubscriptionRecordHandler.SubscriptionResultSet) resultSets.get(0); + } + + private static java.util.Map> tabletsWithDatabase(final Tablet tablet) { + return Collections.singletonMap("root.sg", Collections.singletonList(tablet)); + } + + private static Tablet tablet() { + return tablet("table1"); + } + + private static Tablet tablet(final String tableName) { + final Tablet tablet = + new Tablet( + tableName, + Arrays.asList("s1", "id"), + Arrays.asList(TSDataType.INT64, TSDataType.STRING), + Arrays.asList( + org.apache.tsfile.enums.ColumnCategory.FIELD, + org.apache.tsfile.enums.ColumnCategory.TAG), + 1); + tablet.addTimestamp(0, 1234L); + tablet.addValue("s1", 0, 100L); + tablet.addValue("id", 0, "deviceA"); + return tablet; + } +} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/subscription/SubscriptionInfo.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/subscription/SubscriptionInfo.java index a013b3d7f920e..4359fdcf892a5 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/subscription/SubscriptionInfo.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/subscription/SubscriptionInfo.java @@ -74,8 +74,6 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.function.Predicate; -import java.util.regex.Pattern; -import java.util.regex.PatternSyntaxException; import java.util.stream.Collectors; import java.util.stream.StreamSupport; @@ -333,7 +331,7 @@ private void validateTopicConfig(final TopicConfig topicConfig) throws Subscript throw new SubscriptionException(exceptionMessage); } - validateConsensusTableColumnPattern(topicConfig); + validateColumnFilter(topicConfig); validateConsensusTopicRetentionConfig(topicConfig); final Long ownerLeaseDurationMs = @@ -374,41 +372,41 @@ private void validateConsensusProtocolSupport(final TopicConfig topicConfig) throw new SubscriptionException(exceptionMessage); } - private void validateConsensusTableColumnPattern(final TopicConfig topicConfig) - throws SubscriptionException { - if (!topicConfig.hasAttribute(TopicConstant.COLUMN_KEY)) { - return; + private void validateColumnFilter(final TopicConfig topicConfig) throws SubscriptionException { + int columnFilterKeyCount = 0; + for (final String key : topicConfig.getAttribute().keySet()) { + if (TopicConstant.COLUMN_FILTER_KEY.equalsIgnoreCase(key)) { + columnFilterKeyCount++; + } } - - if (!topicConfig.isTableTopic()) { + if (columnFilterKeyCount > 1) { final String exceptionMessage = String.format( - "Failed to create or alter topic, %s is only supported for table topics", - TopicConstant.COLUMN_KEY); + "Failed to create or alter topic, duplicate %s attributes are not allowed", + TopicConstant.COLUMN_FILTER_KEY); LOGGER.warn(exceptionMessage); throw new SubscriptionException(exceptionMessage); } - if (!isConsensusBasedTopicConfig(topicConfig)) { + if (!topicConfig.hasColumnFilter()) { + return; + } + + if (!topicConfig.isTableTopic()) { final String exceptionMessage = String.format( - "Failed to create or alter topic, %s is only supported for consensus table topics", - TopicConstant.COLUMN_KEY); + "Failed to create or alter topic, %s is only supported for table topics", + TopicConstant.COLUMN_FILTER_KEY); LOGGER.warn(exceptionMessage); throw new SubscriptionException(exceptionMessage); } - final String columnPattern = - topicConfig.getStringOrDefault( - TopicConstant.COLUMN_KEY, TopicConstant.COLUMN_DEFAULT_VALUE); - try { - Pattern.compile(columnPattern); - } catch (final PatternSyntaxException e) { + if (topicConfig.getColumnFilter().trim().isEmpty()) { final String exceptionMessage = String.format( - "Failed to create or alter topic, illegal %s=%s, detail: %s", - TopicConstant.COLUMN_KEY, columnPattern, e.getMessage()); - LOGGER.warn(exceptionMessage, e); + "Failed to create or alter topic, %s should not be empty", + TopicConstant.COLUMN_FILTER_KEY); + LOGGER.warn(exceptionMessage); throw new SubscriptionException(exceptionMessage); } } @@ -479,21 +477,6 @@ private void validateUnsupportedHotUpdatedTopicConfig( throw new SubscriptionException(exceptionMessage); } - final String existedColumnPattern = - existedConfig.getStringOrDefault( - TopicConstant.COLUMN_KEY, TopicConstant.COLUMN_DEFAULT_VALUE); - final String updatedColumnPattern = - updatedConfig.getStringOrDefault( - TopicConstant.COLUMN_KEY, TopicConstant.COLUMN_DEFAULT_VALUE); - if (!Objects.equals(existedColumnPattern, updatedColumnPattern)) { - final String exceptionMessage = - String.format( - "Failed to alter topic %s, changing %s is not supported because existing consensus queues do not hot-refresh converter state", - topicName, TopicConstant.COLUMN_KEY); - LOGGER.warn(exceptionMessage); - throw new SubscriptionException(exceptionMessage); - } - validateUnsupportedHotUpdatedRetentionConfig( topicName, existedConfig, diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/subscription/SubscriptionInfoTopicValidationTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/subscription/SubscriptionInfoTopicValidationTest.java index 61eaa8a4c4c64..c63e4fbd27617 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/subscription/SubscriptionInfoTopicValidationTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/subscription/SubscriptionInfoTopicValidationTest.java @@ -36,10 +36,10 @@ public class SubscriptionInfoTopicValidationTest { @Test - public void testValidateConsensusTableColumnPatternOnCreate() throws Exception { + public void testValidateColumnFilterOnCreate() throws Exception { final SubscriptionInfo subscriptionInfo = new SubscriptionInfo(); final Map attributes = newConsensusTableTopicAttributes(); - attributes.put(TopicConstant.COLUMN_KEY, "(id1|m1)"); + attributes.put(TopicConstant.COLUMN_FILTER_KEY, "column_name IN (\"id1\", \"m1\")"); Assert.assertTrue( subscriptionInfo.validateBeforeCreatingTopic( @@ -47,22 +47,54 @@ public void testValidateConsensusTableColumnPatternOnCreate() throws Exception { } @Test - public void testRejectColumnPatternOnTreeTopic() { + public void testRejectColumnFilterOnTreeTopic() { final SubscriptionInfo subscriptionInfo = new SubscriptionInfo(); final Map attributes = new HashMap<>(); - attributes.put(TopicConstant.COLUMN_KEY, "id1"); + attributes.put(TopicConstant.COLUMN_FILTER_KEY, "column_name = \"id1\""); assertCreateRejected(subscriptionInfo, attributes, "only supported for table topics"); } @Test - public void testRejectColumnPatternOnTsFileTopic() { + public void testColumnFilterKeyIsCaseInsensitiveOnCreate() throws Exception { final SubscriptionInfo subscriptionInfo = new SubscriptionInfo(); - final Map attributes = newConsensusTableTopicAttributes(); + final Map attributes = newLiveTableTopicAttributes(); + attributes.put("Column-Filter", "column_name = \"id1\""); + + Assert.assertTrue( + subscriptionInfo.validateBeforeCreatingTopic( + new TCreateTopicReq("table_topic").setTopicAttributes(attributes))); + } + + @Test + public void testRejectDuplicateColumnFilterKeys() { + final SubscriptionInfo subscriptionInfo = new SubscriptionInfo(); + final Map attributes = newLiveTableTopicAttributes(); + attributes.put(TopicConstant.COLUMN_FILTER_KEY, "column_name = \"id1\""); + attributes.put("Column-Filter", "column_name = \"m1\""); + + assertCreateRejected(subscriptionInfo, attributes, "duplicate column-filter"); + } + + @Test + public void testRejectMixedCaseColumnFilterOnTreeTopic() { + final SubscriptionInfo subscriptionInfo = new SubscriptionInfo(); + final Map attributes = new HashMap<>(); + attributes.put("Column-Filter", "column_name = \"id1\""); + + assertCreateRejected(subscriptionInfo, attributes, "only supported for table topics"); + } + + @Test + public void testAcceptColumnFilterOnLiveTsFileTableTopic() throws Exception { + final SubscriptionInfo subscriptionInfo = new SubscriptionInfo(); + final Map attributes = newLiveTableTopicAttributes(); attributes.put(TopicConstant.FORMAT_KEY, TopicConstant.FORMAT_TS_FILE_VALUE); - attributes.put(TopicConstant.COLUMN_KEY, "id1"); + attributes.put(TopicConstant.COLUMN_FILTER_KEY, "column_name = \"id1\""); - assertCreateRejected(subscriptionInfo, attributes, "mode=consensus only supports format"); + Assert.assertTrue( + subscriptionInfo.validateBeforeCreatingTopic( + new TCreateTopicReq("table_topic").setTopicAttributes(attributes))); } @Test @@ -75,32 +107,27 @@ public void testRejectLegacyTsFileAliasOnConsensusTopic() { } @Test - public void testRejectIllegalColumnRegex() { + public void testRejectEmptyColumnFilter() { final SubscriptionInfo subscriptionInfo = new SubscriptionInfo(); final Map attributes = newConsensusTableTopicAttributes(); - attributes.put(TopicConstant.COLUMN_KEY, "["); + attributes.put(TopicConstant.COLUMN_FILTER_KEY, " "); - assertCreateRejected(subscriptionInfo, attributes, "illegal column"); + assertCreateRejected(subscriptionInfo, attributes, "column-filter should not be empty"); } @Test - public void testRejectAlteringColumnPattern() throws Exception { + public void testAcceptAlteringColumnFilter() throws Exception { final SubscriptionInfo subscriptionInfo = new SubscriptionInfo(); final Map originalAttributes = newConsensusTableTopicAttributes(); - originalAttributes.put(TopicConstant.COLUMN_KEY, "id1"); + originalAttributes.put(TopicConstant.COLUMN_FILTER_KEY, "column_name = \"id1\""); subscriptionInfo.createTopic( new CreateTopicPlan(new TopicMeta("table_topic", 1L, originalAttributes))); final Map updatedAttributes = newConsensusTableTopicAttributes(); - updatedAttributes.put(TopicConstant.COLUMN_KEY, "m1"); + updatedAttributes.put(TopicConstant.COLUMN_FILTER_KEY, "column_name = \"m1\""); - try { - subscriptionInfo.validateBeforeAlteringTopic( - new TopicMeta("table_topic", 2L, updatedAttributes)); - Assert.fail("Expected altering the column pattern to be rejected"); - } catch (final SubscriptionException e) { - Assert.assertTrue(e.getMessage().contains("changing column is not supported")); - } + subscriptionInfo.validateBeforeAlteringTopic( + new TopicMeta("table_topic", 2L, updatedAttributes)); } @Test @@ -173,12 +200,14 @@ public void testRejectIllegalMode() { } @Test - public void testRejectConsensusOnlyColumnOnLiveTopic() { + public void testAcceptColumnFilterOnLiveTableTopic() throws Exception { final SubscriptionInfo subscriptionInfo = new SubscriptionInfo(); final Map attributes = newLiveTableTopicAttributes(); - attributes.put(TopicConstant.COLUMN_KEY, "id1"); + attributes.put(TopicConstant.COLUMN_FILTER_KEY, "column_name = \"id1\""); - assertCreateRejected(subscriptionInfo, attributes, "only supported for consensus table topics"); + Assert.assertTrue( + subscriptionInfo.validateBeforeCreatingTopic( + new TCreateTopicReq("table_topic").setTopicAttributes(attributes))); } @Test diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/batch/PipeTabletEventBatch.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/batch/PipeTabletEventBatch.java index aede0e994d9a9..559307df06f7b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/batch/PipeTabletEventBatch.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/batch/PipeTabletEventBatch.java @@ -95,6 +95,12 @@ public synchronized boolean onEvent(final TabletInsertionEvent event) try { if (constructBatch(event)) { events.add((EnrichedEvent) event); + if (firstEventProcessingTime == Long.MIN_VALUE) { + firstEventProcessingTime = System.currentTimeMillis(); + } + } else { + ((EnrichedEvent) event) + .decreaseReferenceCount(PipeTransferBatchReqBuilder.class.getName(), true); } } catch (final Exception e) { // If the event is not added to the batch, we need to decrease the reference count. @@ -103,10 +109,6 @@ public synchronized boolean onEvent(final TabletInsertionEvent event) // Will cause a retry throw e; } - - if (firstEventProcessingTime == Long.MIN_VALUE) { - firstEventProcessingTime = System.currentTimeMillis(); - } } else { LOGGER.warn(DataNodePipeMessages.CANNOT_INCREASE_REFERENCE_COUNT_FOR_EVENT_IGNORE, event); } @@ -119,8 +121,8 @@ public synchronized boolean onEvent(final TabletInsertionEvent event) * Added an {@link TabletInsertionEvent} into batch. * * @param event the {@link TabletInsertionEvent} in batch - * @return {@code true} if the event is calculated into batch, {@code false} if the event is - * cached and not emitted in this batch. If there are failure encountered, just throw + * @return {@code true} if the event is retained by this batch, {@code false} if the event is + * consumed but produces no batched payload. If there are failure encountered, just throw * exceptions and do not return {@code false} here. */ protected abstract boolean constructBatch(final TabletInsertionEvent event) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/batch/PipeTabletEventTsFileBatch.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/batch/PipeTabletEventTsFileBatch.java index 7b511e23fc6c9..3a9d5a3a46616 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/batch/PipeTabletEventTsFileBatch.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/batch/PipeTabletEventTsFileBatch.java @@ -45,6 +45,7 @@ import java.util.Map; import java.util.Objects; import java.util.concurrent.atomic.AtomicLong; +import java.util.function.BiFunction; import static org.apache.iotdb.db.pipe.event.common.tablet.PipeRawTabletInsertionEvent.isTabletEmpty; @@ -57,30 +58,44 @@ public class PipeTabletEventTsFileBatch extends PipeTabletEventBatch { private final PipeTsFileBuilder treeModeTsFileBuilder; private final PipeTsFileBuilder tableModeTsFileBuilder; + private final BiFunction tableModelTabletPruner; private final Map, Double> pipeName2WeightMap = new HashMap<>(); public PipeTabletEventTsFileBatch(final int maxDelayInMs, final long requestMaxBatchSizeInBytes) { - super(maxDelayInMs, requestMaxBatchSizeInBytes, null); - - final AtomicLong tsFileIdGenerator = new AtomicLong(0); - treeModeTsFileBuilder = new PipeTreeModelTsFileBuilderV2(currentBatchId, tsFileIdGenerator); - tableModeTsFileBuilder = new PipeTableModelTsFileBuilderV2(currentBatchId, tsFileIdGenerator); + this(maxDelayInMs, requestMaxBatchSizeInBytes, null, null); } public PipeTabletEventTsFileBatch( final int maxDelayInMs, final long requestMaxBatchSizeInBytes, final TriLongConsumer recordMetric) { + this(maxDelayInMs, requestMaxBatchSizeInBytes, recordMetric, null); + } + + public PipeTabletEventTsFileBatch( + final int maxDelayInMs, + final long requestMaxBatchSizeInBytes, + final BiFunction tableModelTabletPruner) { + this(maxDelayInMs, requestMaxBatchSizeInBytes, null, tableModelTabletPruner); + } + + public PipeTabletEventTsFileBatch( + final int maxDelayInMs, + final long requestMaxBatchSizeInBytes, + final TriLongConsumer recordMetric, + final BiFunction tableModelTabletPruner) { super(maxDelayInMs, requestMaxBatchSizeInBytes, recordMetric); final AtomicLong tsFileIdGenerator = new AtomicLong(0); treeModeTsFileBuilder = new PipeTreeModelTsFileBuilderV2(currentBatchId, tsFileIdGenerator); tableModeTsFileBuilder = new PipeTableModelTsFileBuilderV2(currentBatchId, tsFileIdGenerator); + this.tableModelTabletPruner = tableModelTabletPruner; } @Override protected boolean constructBatch(final TabletInsertionEvent event) { + boolean hasBufferedTablet = false; if (event instanceof PipeInsertNodeTabletInsertionEvent) { final PipeInsertNodeTabletInsertionEvent insertNodeTabletInsertionEvent = (PipeInsertNodeTabletInsertionEvent) event; @@ -93,11 +108,18 @@ protected boolean constructBatch(final TabletInsertionEvent event) { } if (isTableModel) { // table Model + final Tablet prunedTablet = + pruneTableModelTablet( + tablet, insertNodeTabletInsertionEvent.getTableModelDatabaseName()); + if (isTabletEmpty(prunedTablet)) { + continue; + } bufferTableModelTablet( insertNodeTabletInsertionEvent.getPipeName(), insertNodeTabletInsertionEvent.getCreationTime(), - tablet, + prunedTablet, insertNodeTabletInsertionEvent.getTableModelDatabaseName()); + hasBufferedTablet = true; } else { // tree Model bufferTreeModelTablet( @@ -105,6 +127,7 @@ protected boolean constructBatch(final TabletInsertionEvent event) { insertNodeTabletInsertionEvent.getCreationTime(), tablet, insertNodeTabletInsertionEvent.isAligned(i)); + hasBufferedTablet = true; } } } else if (event instanceof PipeRawTabletInsertionEvent) { @@ -112,15 +135,21 @@ protected boolean constructBatch(final TabletInsertionEvent event) { (PipeRawTabletInsertionEvent) event; final Tablet tablet = rawTabletInsertionEvent.convertToTablet(); if (isTabletEmpty(tablet)) { - return true; + return false; } if (rawTabletInsertionEvent.isTableModelEvent()) { // table Model + final Tablet prunedTablet = + pruneTableModelTablet(tablet, rawTabletInsertionEvent.getTableModelDatabaseName()); + if (isTabletEmpty(prunedTablet)) { + return false; + } bufferTableModelTablet( rawTabletInsertionEvent.getPipeName(), rawTabletInsertionEvent.getCreationTime(), - tablet, + prunedTablet, rawTabletInsertionEvent.getTableModelDatabaseName()); + hasBufferedTablet = true; } else { // tree Model bufferTreeModelTablet( @@ -128,6 +157,7 @@ protected boolean constructBatch(final TabletInsertionEvent event) { rawTabletInsertionEvent.getCreationTime(), tablet, rawTabletInsertionEvent.isAligned()); + hasBufferedTablet = true; } } else { LOGGER.warn( @@ -136,7 +166,13 @@ protected boolean constructBatch(final TabletInsertionEvent event) { event, event.getClass()); } - return true; + return hasBufferedTablet; + } + + private Tablet pruneTableModelTablet(final Tablet tablet, final String databaseName) { + return Objects.nonNull(tableModelTabletPruner) + ? tableModelTabletPruner.apply(databaseName, tablet) + : tablet; } private void bufferTreeModelTablet( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java index 4eff93bfc12d2..c9c56061feace 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java @@ -254,8 +254,11 @@ import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowConfigurationStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.StartRepairDataStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.StopRepairDataStatement; +import org.apache.iotdb.db.subscription.columnfilter.ColumnFilterParser; import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters; import org.apache.iotdb.rpc.TSStatusCode; +import org.apache.iotdb.rpc.subscription.config.TopicConstant; +import org.apache.iotdb.rpc.subscription.exception.SubscriptionException; import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.enums.TSDataType; @@ -295,6 +298,8 @@ public class TableConfigTaskVisitor implements AstVisitor topicAttributes) { + String columnFilterKey = null; + String columnFilter = null; + boolean hasColumnFilter = false; + for (final Map.Entry entry : topicAttributes.entrySet()) { + if (TopicConstant.COLUMN_FILTER_KEY.equalsIgnoreCase(entry.getKey())) { + if (hasColumnFilter) { + throw new SemanticException( + String.format( + "Failed to create or alter topic, duplicate %s attributes are not allowed", + TopicConstant.COLUMN_FILTER_KEY)); + } + hasColumnFilter = true; + columnFilterKey = entry.getKey(); + columnFilter = entry.getValue(); + } + } + + if (!hasColumnFilter) { + return; + } + + if (!TopicConstant.COLUMN_FILTER_KEY.equals(columnFilterKey)) { + topicAttributes.remove(columnFilterKey); + topicAttributes.put(TopicConstant.COLUMN_FILTER_KEY, columnFilter); + } + + try { + COLUMN_FILTER_PARSER.parseAndValidate(columnFilter); + } catch (final SubscriptionException e) { + throw new SemanticException(e.getMessage()); + } + } + @Override public IConfigTask visitDropTopic(DropTopic node, MPPQueryContext context) { context.setQueryType(QueryType.OTHER); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TreeConfigTaskVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TreeConfigTaskVisitor.java index 28745712d220c..600c7f0cdce2a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TreeConfigTaskVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TreeConfigTaskVisitor.java @@ -232,6 +232,7 @@ import org.apache.iotdb.db.queryengine.plan.statement.sys.quota.ShowSpaceQuotaStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.quota.ShowThrottleQuotaStatement; import org.apache.iotdb.rpc.TSStatusCode; +import org.apache.iotdb.rpc.subscription.config.TopicConstant; import org.apache.tsfile.exception.NotImplementedException; @@ -728,6 +729,7 @@ public IConfigTask visitCreateTopic( createTopicStatement .getTopicAttributes() .put(SystemConstant.SQL_DIALECT_KEY, SystemConstant.SQL_DIALECT_TREE_VALUE); + rejectColumnFilterForTreeTopic(createTopicStatement.getTopicAttributes()); return new CreateTopicTask(createTopicStatement); } @@ -738,10 +740,22 @@ public IConfigTask visitAlterTopic( alterTopicStatement .getTopicAttributes() .put(SystemConstant.SQL_DIALECT_KEY, SystemConstant.SQL_DIALECT_TREE_VALUE); + rejectColumnFilterForTreeTopic(alterTopicStatement.getTopicAttributes()); return new AlterTopicTask(alterTopicStatement); } + private static void rejectColumnFilterForTreeTopic(final Map topicAttributes) { + for (final String key : topicAttributes.keySet()) { + if (TopicConstant.COLUMN_FILTER_KEY.equalsIgnoreCase(key)) { + throw new SemanticException( + String.format( + "Failed to create or alter topic, %s is only supported for table topics", + TopicConstant.COLUMN_FILTER_KEY)); + } + } + } + @Override public IConfigTask visitDropTopic( DropTopicStatement dropTopicStatement, MPPQueryContext context) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCache.java index f545a9bda237d..9485e7c5a0547 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCache.java @@ -313,6 +313,28 @@ public long getInstanceVersion() { return instanceVersion.get(); } + public Map> getTableSnapshot() { + readWriteLock.readLock().lock(); + try { + return databaseTableMap.entrySet().stream() + .collect( + Collectors.toMap( + Map.Entry::getKey, + entry -> + entry.getValue().entrySet().stream() + .collect( + Collectors.toMap( + Map.Entry::getKey, + tableEntry -> new TsTable(tableEntry.getValue()), + (left, right) -> right, + ConcurrentHashMap::new)), + (left, right) -> right, + ConcurrentHashMap::new)); + } finally { + readWriteLock.readLock().unlock(); + } + } + public TsTable getTableInWrite(final String database, final String tableName) { final TsTable result = getTableInCache(database, tableName); return Objects.nonNull(result) ? result : getTable(database, tableName, false); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionBrokerAgent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionBrokerAgent.java index e177f02746ab3..6ce982409493b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionBrokerAgent.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionBrokerAgent.java @@ -28,6 +28,7 @@ import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.consensus.DataRegionConsensusImpl; import org.apache.iotdb.db.i18n.DataNodeMiscMessages; +import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache; import org.apache.iotdb.db.subscription.broker.ConsensusSubscriptionBroker; import org.apache.iotdb.db.subscription.broker.ISubscriptionBroker; import org.apache.iotdb.db.subscription.broker.SubscriptionBroker; @@ -35,11 +36,14 @@ import org.apache.iotdb.db.subscription.broker.consensus.ConsensusRegionRuntimeState; import org.apache.iotdb.db.subscription.broker.consensus.ConsensusSubscriptionCommitManager; import org.apache.iotdb.db.subscription.broker.consensus.ConsensusSubscriptionSetupHandler; +import org.apache.iotdb.db.subscription.columnfilter.ColumnFilterBinder; +import org.apache.iotdb.db.subscription.columnfilter.ColumnFilterMatcher; import org.apache.iotdb.db.subscription.event.SubscriptionEvent; import org.apache.iotdb.db.subscription.resource.SubscriptionDataNodeResourceManager; import org.apache.iotdb.db.subscription.task.execution.ConsensusSubscriptionPrefetchExecutorManager; import org.apache.iotdb.db.subscription.task.subtask.SubscriptionSinkSubtask; import org.apache.iotdb.rpc.subscription.config.ConsumerConfig; +import org.apache.iotdb.rpc.subscription.config.TopicConfig; import org.apache.iotdb.rpc.subscription.exception.SubscriptionException; import org.apache.iotdb.rpc.subscription.payload.poll.RegionProgress; import org.apache.iotdb.rpc.subscription.payload.poll.SubscriptionCommitContext; @@ -76,6 +80,10 @@ public class SubscriptionBrokerAgent { private final Cache prefetchingQueueCount = new Cache<>(this::getPrefetchingQueueCountInternal); + private final Map topicNameToColumnFilterMatcher = + new ConcurrentHashMap<>(); + private final ColumnFilterBinder columnFilterBinder = new ColumnFilterBinder(); + //////////////////////////// provided for subscription agent //////////////////////////// public List poll( @@ -649,6 +657,56 @@ public void refreshConsensusQueueOrderMode(final String topicName, final String } } + public void refreshColumnFilter(final String topicName, final TopicConfig topicConfig) { + final ColumnFilterMatcher matcher; + try { + matcher = + ColumnFilterMatcher.fromBoundColumnFilter( + columnFilterBinder.bind( + topicConfig, DataNodeTableCache.getInstance().getTableSnapshot())); + } catch (final Exception e) { + LOGGER.warn( + "SubscriptionBrokerAgent: failed to refresh column-filter matcher for topic [{}], use empty matcher to fail closed", + topicName, + e); + topicNameToColumnFilterMatcher.put( + topicName, ColumnFilterMatcher.ofSelectedColumnNames(Collections.emptySet())); + return; + } + topicNameToColumnFilterMatcher.put(topicName, matcher); + LOGGER.info( + "SubscriptionBrokerAgent: refreshed column-filter matcher for topic [{}]", topicName); + } + + public ColumnFilterMatcher getColumnFilterMatcher(final String topicName) { + final ColumnFilterMatcher matcher = topicNameToColumnFilterMatcher.get(topicName); + if (Objects.nonNull(matcher)) { + return matcher; + } + + final TopicConfig topicConfig = + SubscriptionAgent.topic().getTopicConfigs(Collections.singleton(topicName)).get(topicName); + if (Objects.isNull(topicConfig)) { + return ColumnFilterMatcher.matchAll(); + } + + try { + refreshColumnFilter(topicName, topicConfig); + return topicNameToColumnFilterMatcher.getOrDefault(topicName, ColumnFilterMatcher.matchAll()); + } catch (final Exception e) { + LOGGER.warn( + "SubscriptionBrokerAgent: failed to lazily refresh column-filter matcher for topic [{}]", + topicName, + e); + return ColumnFilterMatcher.ofSelectedColumnNames(Collections.emptySet()); + } + } + + public void dropColumnFilter(final String topicName) { + topicNameToColumnFilterMatcher.remove(topicName); + LOGGER.info("SubscriptionBrokerAgent: dropped column-filter matcher for topic [{}]", topicName); + } + public void unbindConsensusPrefetchingQueue( final String consumerGroupId, final String topicName) { final ConsensusSubscriptionBroker broker = getConsensusBroker(consumerGroupId); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionTopicAgent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionTopicAgent.java index 485e553e8ee0a..9668eedf16ceb 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionTopicAgent.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionTopicAgent.java @@ -35,6 +35,7 @@ import org.slf4j.LoggerFactory; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.Set; @@ -93,14 +94,62 @@ public TPushTopicMetaRespExceptionMessage handleSingleTopicMetaChanges( private void handleSingleTopicMetaChangesInternal(final TopicMeta metaFromCoordinator) { final String topicName = metaFromCoordinator.getTopicName(); - TopicMeta.validateOwnerProgression( - topicMetaKeeper.getTopicMeta(topicName), metaFromCoordinator); + final TopicMeta oldMeta = topicMetaKeeper.getTopicMeta(topicName); + TopicMeta.validateOwnerProgression(oldMeta, metaFromCoordinator); topicMetaKeeper.removeTopicMeta(topicName); topicMetaKeeper.addTopicMeta(topicName, metaFromCoordinator); + if (shouldRefreshColumnFilter(oldMeta, metaFromCoordinator)) { + SubscriptionAgent.broker().refreshColumnFilter(topicName, metaFromCoordinator.getConfig()); + } else if (!metaFromCoordinator.getConfig().isTableTopic()) { + SubscriptionAgent.broker().dropColumnFilter(topicName); + } SubscriptionAgent.broker() .refreshConsensusQueueOrderMode(topicName, metaFromCoordinator.getConfig().getOrderMode()); } + static boolean shouldRefreshColumnFilter(final TopicMeta oldMeta, final TopicMeta newMeta) { + if (Objects.isNull(newMeta) || !newMeta.getConfig().isTableTopic()) { + return false; + } + if (Objects.isNull(oldMeta) || !oldMeta.getConfig().isTableTopic()) { + return true; + } + + final TopicConfig oldConfig = oldMeta.getConfig(); + final TopicConfig newConfig = newMeta.getConfig(); + return !Objects.equals( + normalizeColumnFilterBindingValue(oldConfig.getColumnFilter()), + normalizeColumnFilterBindingValue(newConfig.getColumnFilter())) + || !Objects.equals( + normalizeColumnFilterBindingValue( + getAttributeIgnoreCase( + oldConfig, TopicConstant.DATABASE_KEY, TopicConstant.DATABASE_DEFAULT_VALUE)), + normalizeColumnFilterBindingValue( + getAttributeIgnoreCase( + newConfig, TopicConstant.DATABASE_KEY, TopicConstant.DATABASE_DEFAULT_VALUE))) + || !Objects.equals( + normalizeColumnFilterBindingValue( + getAttributeIgnoreCase( + oldConfig, TopicConstant.TABLE_KEY, TopicConstant.TABLE_DEFAULT_VALUE)), + normalizeColumnFilterBindingValue( + getAttributeIgnoreCase( + newConfig, TopicConstant.TABLE_KEY, TopicConstant.TABLE_DEFAULT_VALUE))); + } + + private static String getAttributeIgnoreCase( + final TopicConfig topicConfig, final String key, final String defaultValue) { + return topicConfig.getAttribute().entrySet().stream() + .filter(entry -> key.equalsIgnoreCase(entry.getKey())) + .map(Map.Entry::getValue) + .filter(Objects::nonNull) + .findFirst() + .orElse(defaultValue); + } + + private static String normalizeColumnFilterBindingValue(final String value) { + return Objects.nonNull(value) ? value.trim().toLowerCase(Locale.ROOT) : ""; + } + public TPushTopicMetaRespExceptionMessage handleTopicMetaChanges( final List topicMetasFromCoordinator) { acquireWriteLock(); @@ -146,6 +195,7 @@ public TPushTopicMetaRespExceptionMessage handleDropTopic(final String topicName private void handleDropTopicInternal(final String topicName) { topicMetaKeeper.removeTopicMeta(topicName); + SubscriptionAgent.broker().dropColumnFilter(topicName); } public boolean isTopicExisted(final String topicName) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionPrefetchingQueue.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionPrefetchingQueue.java index 086fde0fbcc9e..13aac581a4e41 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionPrefetchingQueue.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionPrefetchingQueue.java @@ -147,6 +147,10 @@ public void cleanUp() { } } + public String getTopicName() { + return topicName; + } + protected void cleanUpInternal() { // clean up events in batches batches.cleanUp(); @@ -633,7 +637,8 @@ private synchronized void tryPrefetchV2() { } if (event instanceof TsFileInsertionEvent) { - if (PipeEventCollector.canSkipParsing4TsFileEvent((PipeTsFileInsertionEvent) event)) { + final PipeTsFileInsertionEvent pipeTsFileInsertionEvent = (PipeTsFileInsertionEvent) event; + if (canPassThroughTsFile(pipeTsFileInsertionEvent)) { onEvent((TsFileInsertionEvent) event); return; } @@ -643,7 +648,7 @@ private synchronized void tryPrefetchV2() { this, event); } else { - constructToTabletIterator((PipeTsFileInsertionEvent) event); + constructToTabletIterator(pipeTsFileInsertionEvent); return; } } @@ -676,6 +681,12 @@ private void constructToTabletIterator(final TsFileInsertionEvent event) { } } + private boolean canPassThroughTsFile(final PipeTsFileInsertionEvent event) { + return PipeEventCollector.canSkipParsing4TsFileEvent(event) + && (!event.isTableModelEvent() + || SubscriptionAgent.broker().getColumnFilterMatcher(topicName).isMatchAll()); + } + private RetryableState onRetryableTabletInsertionEvent( final RetryableEvent retryableEvent) { currentTabletInsertionEvent = null; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusLogToTabletConverter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusLogToTabletConverter.java index 300c3c792e940..cfb731f76d708 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusLogToTabletConverter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusLogToTabletConverter.java @@ -38,6 +38,9 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertTabletNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.SearchNode; import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.WALEntry; +import org.apache.iotdb.db.subscription.agent.SubscriptionAgent; +import org.apache.iotdb.db.subscription.columnfilter.ColumnFilterMatcher; +import org.apache.iotdb.db.subscription.columnfilter.TabletColumnPruner; import org.apache.tsfile.enums.ColumnCategory; import org.apache.tsfile.enums.TSDataType; @@ -55,7 +58,6 @@ import java.util.Collections; import java.util.List; import java.util.Objects; -import java.util.regex.Pattern; /** Converts IoTConsensus WAL log entries (InsertNode) to Tablet format for subscription. */ public class ConsensusLogToTabletConverter { @@ -64,7 +66,8 @@ public class ConsensusLogToTabletConverter { private final TreePattern treePattern; private final TablePattern tablePattern; - private final Pattern tableColumnPattern; + private final String topicName; + private final ColumnFilterMatcher fallbackColumnFilterMatcher; /** * The actual database name of the DataRegion this converter processes (table-model format without @@ -75,11 +78,22 @@ public class ConsensusLogToTabletConverter { public ConsensusLogToTabletConverter( final TreePattern treePattern, final TablePattern tablePattern, - final Pattern tableColumnPattern, + final ColumnFilterMatcher columnFilterMatcher, + final String databaseName) { + this(treePattern, tablePattern, null, columnFilterMatcher, databaseName); + } + + public ConsensusLogToTabletConverter( + final TreePattern treePattern, + final TablePattern tablePattern, + final String topicName, + final ColumnFilterMatcher columnFilterMatcher, final String databaseName) { this.treePattern = treePattern; this.tablePattern = tablePattern; - this.tableColumnPattern = tableColumnPattern; + this.topicName = topicName; + this.fallbackColumnFilterMatcher = + Objects.nonNull(columnFilterMatcher) ? columnFilterMatcher : ColumnFilterMatcher.matchAll(); this.databaseName = databaseName; } @@ -458,43 +472,18 @@ private List convertRelationalInsertRowNode(final RelationalInsertRowNod final String[] measurements = node.getMeasurements(); final TSDataType[] dataTypes = node.getDataTypes(); final Object[] values = node.getValues(); - final List matchedColumnIndices = - getMatchedTableColumnIndices( - measurements, dataTypes, values, node.getColumnCategories(), false); - if (matchedColumnIndices.isEmpty()) { - return Collections.emptyList(); - } - - final int columnCount = matchedColumnIndices.size(); - final List columnNames = new ArrayList<>(columnCount); - final List columnDataTypes = new ArrayList<>(columnCount); - final List columnTypes = new ArrayList<>(columnCount); - for (final int originalColIdx : matchedColumnIndices) { - columnNames.add(measurements[originalColIdx]); - columnDataTypes.add(dataTypes[originalColIdx]); - columnTypes.add(toTsFileColumnCategory(node.getColumnCategories(), originalColIdx)); - } - final Tablet tablet = - new Tablet( - tableName != null ? tableName : "", columnNames, columnDataTypes, columnTypes, 1); - tablet.addTimestamp(0, time); - - for (int i = 0; i < columnCount; i++) { - final int originalColIdx = matchedColumnIndices.get(i); - final Object value = values[originalColIdx]; - if (value == null) { - if (tablet.getBitMaps() == null) { - tablet.initBitMaps(); - } - tablet.getBitMaps()[i].mark(0); - } else { - addValueToTablet(tablet, 0, i, dataTypes[originalColIdx], value); - } + buildTableModelTabletFromRow( + tableName, time, measurements, dataTypes, values, node.getColumnCategories()); + if (Objects.isNull(tablet)) { + return Collections.emptyList(); } - tablet.setRowSize(1); - return Collections.singletonList(tablet); + final Tablet prunedTablet = + TabletColumnPruner.pruneTableModelTablet(tablet, databaseName, getColumnFilterMatcher()); + return Objects.nonNull(prunedTablet) + ? Collections.singletonList(prunedTablet) + : Collections.emptyList(); } private List convertRelationalInsertTabletNode(final RelationalInsertTabletNode node) { @@ -512,51 +501,32 @@ private List convertRelationalInsertTabletNode(final RelationalInsertTab final String[] measurements = node.getMeasurements(); final TSDataType[] dataTypes = node.getDataTypes(); - final long[] times = node.getTimes(); - final Object[] columns = node.getColumns(); - final BitMap[] bitMaps = node.getBitMaps(); - final int rowCount = node.getRowCount(); - final List matchedColumnIndices = - getMatchedTableColumnIndices( - measurements, dataTypes, columns, node.getColumnCategories(), true); - if (matchedColumnIndices.isEmpty()) { + if (Objects.isNull(measurements) + || Objects.isNull(dataTypes) + || Objects.isNull(node.getColumns())) { return Collections.emptyList(); } - - final int columnCount = matchedColumnIndices.size(); - final boolean allColumnsMatch = columnCount == measurements.length; - final List schemas = new ArrayList<>(columnCount); - final List columnTypes = new ArrayList<>(columnCount); - for (final int originalColIdx : matchedColumnIndices) { - schemas.add(new MeasurementSchema(measurements[originalColIdx], dataTypes[originalColIdx])); - columnTypes.add(toTsFileColumnCategory(node.getColumnCategories(), originalColIdx)); - } - - // Column filtering changes only the tablet shape. The selected value arrays come from WAL - // InsertNodes and are reused by the subscription read path. - final long[] newTimes = times; - final Object[] newColumns = new Object[columnCount]; - final BitMap[] newBitMaps = new BitMap[columnCount]; - - for (int colIdx = 0; colIdx < columnCount; colIdx++) { - final int originalColIdx = allColumnsMatch ? colIdx : matchedColumnIndices.get(colIdx); - newColumns[colIdx] = columns[originalColIdx]; - if (bitMaps != null && bitMaps[originalColIdx] != null) { - newBitMaps[colIdx] = bitMaps[originalColIdx]; - } + final List schemas = new ArrayList<>(measurements.length); + final List columnTypes = new ArrayList<>(measurements.length); + for (int i = 0; i < measurements.length; i++) { + schemas.add(new MeasurementSchema(measurements[i], dataTypes[i])); + columnTypes.add(toTsFileColumnCategory(node.getColumnCategories(), i)); } - final Tablet tablet = new Tablet( tableName != null ? tableName : "", schemas, columnTypes, - newTimes, - newColumns, - newBitMaps, - rowCount); - - return Collections.singletonList(tablet); + node.getTimes(), + node.getColumns(), + node.getBitMaps(), + node.getRowCount()); + + final Tablet prunedTablet = + TabletColumnPruner.pruneTableModelTablet(tablet, databaseName, getColumnFilterMatcher()); + return Objects.nonNull(prunedTablet) + ? Collections.singletonList(prunedTablet) + : Collections.emptyList(); } private List convertRelationalInsertRowsNode(final RelationalInsertRowsNode node) { @@ -567,6 +537,58 @@ private List convertRelationalInsertRowsNode(final RelationalInsertRowsN return tablets; } + private Tablet buildTableModelTabletFromRow( + final String tableName, + final long time, + final String[] measurements, + final TSDataType[] dataTypes, + final Object[] values, + final TsTableColumnCategory[] columnCategories) { + if (Objects.isNull(measurements) || Objects.isNull(dataTypes) || Objects.isNull(values)) { + return null; + } + + final List columnNames = new ArrayList<>(measurements.length); + final List columnDataTypes = new ArrayList<>(measurements.length); + final List columnTypes = new ArrayList<>(measurements.length); + final List originalColumnIndexes = new ArrayList<>(measurements.length); + for (int i = 0; i < measurements.length && i < dataTypes.length && i < values.length; i++) { + if (Objects.isNull(measurements[i]) || Objects.isNull(dataTypes[i])) { + continue; + } + columnNames.add(measurements[i]); + columnDataTypes.add(dataTypes[i]); + columnTypes.add(toTsFileColumnCategory(columnCategories, i)); + originalColumnIndexes.add(i); + } + if (columnNames.isEmpty()) { + return null; + } + + final Tablet tablet = + new Tablet( + Objects.nonNull(tableName) ? tableName : "", + columnNames, + columnDataTypes, + columnTypes, + 1); + tablet.addTimestamp(0, time); + for (int i = 0; i < originalColumnIndexes.size(); i++) { + final int originalColumnIndex = originalColumnIndexes.get(i); + final Object value = values[originalColumnIndex]; + if (Objects.isNull(value)) { + if (Objects.isNull(tablet.getBitMaps())) { + tablet.initBitMaps(); + } + tablet.getBitMaps()[i].mark(0); + } else { + addValueToTablet(tablet, 0, i, dataTypes[originalColumnIndex], value); + } + } + tablet.setRowSize(1); + return tablet; + } + // ======================== Helper Methods ======================== /** @@ -603,59 +625,6 @@ private List getMatchedTreeColumnIndices( return matchedIndices; } - /** - * Returns indices of table columns to emit. If any column matches the configured column pattern, - * all TAG columns are also kept so consumers can reconstruct the row-level table-model device ID. - * If no table column pattern is specified, all non-null columns are returned. - */ - private List getMatchedTableColumnIndices( - final String[] measurements, - final TSDataType[] dataTypes, - final Object[] valuesOrColumns, - final TsTableColumnCategory[] columnCategories, - final boolean requireNonNullValue) { - if (measurements == null) { - return Collections.emptyList(); - } - final boolean[] selectedColumns = new boolean[measurements.length]; - boolean hasMatchedColumn = false; - for (int i = 0; i < measurements.length; i++) { - if (!isValidColumn(measurements, dataTypes, valuesOrColumns, i, requireNonNullValue)) { - continue; - } - if (tableColumnPattern == null || tableColumnPattern.matcher(measurements[i]).matches()) { - selectedColumns[i] = true; - hasMatchedColumn = true; - } - } - - if (!hasMatchedColumn) { - return Collections.emptyList(); - } - - for (int i = 0; i < measurements.length; i++) { - if (isValidColumn(measurements, dataTypes, valuesOrColumns, i, requireNonNullValue) - && isTagColumn(columnCategories, i)) { - selectedColumns[i] = true; - } - } - - final List matchedIndices = new ArrayList<>(measurements.length); - for (int i = 0; i < selectedColumns.length; i++) { - if (selectedColumns[i]) { - matchedIndices.add(i); - } - } - return matchedIndices; - } - - private boolean isTagColumn( - final TsTableColumnCategory[] columnCategories, final int columnIndex) { - return columnCategories != null - && columnIndex < columnCategories.length - && columnCategories[columnIndex] == TsTableColumnCategory.TAG; - } - private ColumnCategory toTsFileColumnCategory( final TsTableColumnCategory[] columnCategories, final int columnIndex) { return columnCategories != null @@ -665,6 +634,12 @@ private ColumnCategory toTsFileColumnCategory( : ColumnCategory.FIELD; } + private ColumnFilterMatcher getColumnFilterMatcher() { + return Objects.nonNull(topicName) + ? SubscriptionAgent.broker().getColumnFilterMatcher(topicName) + : fallbackColumnFilterMatcher; + } + private boolean isValidColumn( final String[] measurements, final TSDataType[] dataTypes, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusPrefetchingQueue.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusPrefetchingQueue.java index fcd5cc4c3b654..d4dd863170827 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusPrefetchingQueue.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusPrefetchingQueue.java @@ -39,6 +39,8 @@ import org.apache.iotdb.db.storageengine.dataregion.wal.io.WALMetaData; import org.apache.iotdb.db.storageengine.dataregion.wal.node.WALNode; import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALFileUtils; +import org.apache.iotdb.db.subscription.agent.SubscriptionAgent; +import org.apache.iotdb.db.subscription.columnfilter.ColumnFilterMatcher; import org.apache.iotdb.db.subscription.event.SubscriptionEvent; import org.apache.iotdb.db.subscription.metric.ConsensusSubscriptionPrefetchingQueueMetrics; import org.apache.iotdb.db.subscription.task.execution.ConsensusSubscriptionPrefetchExecutor; @@ -1796,7 +1798,11 @@ private boolean createAndEnqueueEvent( final SubscriptionEvent event = new SubscriptionEvent( - SubscriptionPollResponseType.TABLETS.getType(), payload, commitContext); + SubscriptionPollResponseType.TABLETS.getType(), + payload, + commitContext, + SubscriptionAgent.broker().getColumnFilterMatcher(topicName).isTimeSelected(), + getTimeSelectedByTable(converter.getDatabaseName(), tablets)); prefetchingQueue.add(event); @@ -1813,6 +1819,25 @@ private boolean createAndEnqueueEvent( return true; } + private Map> getTimeSelectedByTable( + final String databaseName, final List tablets) { + if (Objects.isNull(databaseName) || Objects.isNull(tablets) || tablets.isEmpty()) { + return Collections.emptyMap(); + } + final ColumnFilterMatcher matcher = + SubscriptionAgent.broker().getColumnFilterMatcher(topicName); + final Map tableMap = new HashMap<>(); + for (final Tablet tablet : tablets) { + if (Objects.nonNull(tablet) && Objects.nonNull(tablet.getTableName())) { + tableMap.put( + tablet.getTableName(), matcher.isTimeSelected(databaseName, tablet.getTableName())); + } + } + return tableMap.isEmpty() + ? Collections.emptyMap() + : Collections.singletonMap(databaseName, tableMap); + } + private SubscriptionCommitContext buildWriterCommitContext(final long localSeq) { final int effectiveNodeId = batchWriterNodeId >= 0 diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusSubscriptionSetupHandler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusSubscriptionSetupHandler.java index 0d072e50f23c1..4cbbec4a8cd43 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusSubscriptionSetupHandler.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusSubscriptionSetupHandler.java @@ -53,7 +53,6 @@ import java.util.Objects; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; -import java.util.regex.Pattern; /** * Handles setup and teardown of consensus-based subscription queues on DataNode. @@ -177,7 +176,8 @@ private static void onNewRegionCreated( } final String actualDbName = topicConfig.isTableTopic() ? dbTableModel : null; - final ConsensusLogToTabletConverter converter = buildConverter(topicConfig, actualDbName); + final ConsensusLogToTabletConverter converter = + buildConverter(topicName, topicConfig, actualDbName); final SubscriptionWalRetentionPolicy retentionPolicy = buildSubscriptionWalRetentionPolicy(topicName, topicConfig, serverImpl); @@ -412,7 +412,8 @@ private static void setupConsensusQueueForTopic( } final String actualDbName = topicConfig.isTableTopic() ? dbTableModel : null; - final ConsensusLogToTabletConverter converter = buildConverter(topicConfig, actualDbName); + final ConsensusLogToTabletConverter converter = + buildConverter(topicName, topicConfig, actualDbName); final SubscriptionWalRetentionPolicy retentionPolicy = buildSubscriptionWalRetentionPolicy(topicName, topicConfig, serverImpl); @@ -487,7 +488,7 @@ private static void setupConsensusQueueForTopic( } private static ConsensusLogToTabletConverter buildConverter( - final TopicConfig topicConfig, final String actualDatabaseName) { + final String topicName, final TopicConfig topicConfig, final String actualDatabaseName) { // Determine tree or table model final boolean isTableTopic = topicConfig.isTableTopic(); @@ -495,15 +496,11 @@ private static ConsensusLogToTabletConverter buildConverter( TablePattern tablePattern = null; if (isTableTopic) { + SubscriptionAgent.broker().refreshColumnFilter(topicName, topicConfig); // Table model: database + table name pattern - final String column = - topicConfig.getStringOrDefault( - TopicConstant.COLUMN_KEY, TopicConstant.COLUMN_DEFAULT_VALUE); tablePattern = buildTablePattern(topicConfig); - final Pattern columnPattern = - TopicConstant.COLUMN_DEFAULT_VALUE.equals(column) ? null : Pattern.compile(column); return new ConsensusLogToTabletConverter( - null, tablePattern, columnPattern, actualDatabaseName); + null, tablePattern, topicName, null, actualDatabaseName); } else { // Tree model: path or pattern if (topicConfig.getAttribute().containsKey(TopicConstant.PATTERN_KEY)) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/columnfilter/BoundColumnFilter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/columnfilter/BoundColumnFilter.java new file mode 100644 index 0000000000000..b15852db5fcf2 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/columnfilter/BoundColumnFilter.java @@ -0,0 +1,146 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.subscription.columnfilter; + +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; +import java.util.Set; + +public class BoundColumnFilter { + + private static final BoundColumnFilter MATCH_ALL = new BoundColumnFilter(true, false, null, null); + + private final boolean matchAll; + private final boolean timeSelected; + private final Map> selectedColumnsByTable; + private final Map timeSelectedByTable; + + private BoundColumnFilter( + final boolean matchAll, + final boolean timeSelected, + final Map> selectedColumnsByTable, + final Map timeSelectedByTable) { + this.matchAll = matchAll; + this.timeSelected = timeSelected; + this.selectedColumnsByTable = selectedColumnsByTable; + this.timeSelectedByTable = timeSelectedByTable; + } + + public static BoundColumnFilter matchAll() { + return MATCH_ALL; + } + + public static BoundColumnFilter of( + final Map> selectedColumnsByTable, + final boolean timeSelected, + final Map timeSelectedByTable) { + final Map> copied = new HashMap<>(); + selectedColumnsByTable.forEach( + (key, value) -> copied.put(key, Collections.unmodifiableSet(new HashSet<>(value)))); + return new BoundColumnFilter( + false, + timeSelected, + Collections.unmodifiableMap(copied), + Objects.nonNull(timeSelectedByTable) + ? Collections.unmodifiableMap(new HashMap<>(timeSelectedByTable)) + : Collections.emptyMap()); + } + + public boolean isMatchAll() { + return matchAll; + } + + public boolean isTimeSelected() { + return timeSelected; + } + + public boolean isTimeSelected(final String databaseName, final String tableName) { + if (matchAll) { + return true; + } + return Boolean.TRUE.equals(timeSelectedByTable.get(TableKey.of(databaseName, tableName))); + } + + public boolean match(final String databaseName, final String tableName, final String columnName) { + if (matchAll) { + return true; + } + final Set selectedColumns = + selectedColumnsByTable.get(TableKey.of(databaseName, tableName)); + return Objects.nonNull(selectedColumns) && selectedColumns.contains(normalize(columnName)); + } + + public Map> getSelectedColumnsByTable() { + return selectedColumnsByTable; + } + + public Map getTimeSelectedByTable() { + return timeSelectedByTable; + } + + static String normalize(final String value) { + return Objects.nonNull(value) ? value.trim().toLowerCase(Locale.ROOT) : ""; + } + + public static final class TableKey { + + private final String databaseName; + private final String tableName; + + private TableKey(final String databaseName, final String tableName) { + this.databaseName = normalize(databaseName); + this.tableName = normalize(tableName); + } + + public static TableKey of(final String databaseName, final String tableName) { + return new TableKey(databaseName, tableName); + } + + public String getDatabaseName() { + return databaseName; + } + + public String getTableName() { + return tableName; + } + + @Override + public boolean equals(final Object o) { + if (this == o) { + return true; + } + if (!(o instanceof TableKey)) { + return false; + } + final TableKey tableKey = (TableKey) o; + return Objects.equals(databaseName, tableKey.databaseName) + && Objects.equals(tableName, tableKey.tableName); + } + + @Override + public int hashCode() { + return Objects.hash(databaseName, tableName); + } + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/columnfilter/ColumnFilterBinder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/columnfilter/ColumnFilterBinder.java new file mode 100644 index 0000000000000..9dd771b38d06a --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/columnfilter/ColumnFilterBinder.java @@ -0,0 +1,166 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.subscription.columnfilter; + +import org.apache.iotdb.commons.pipe.datastructure.pattern.TablePattern; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.Expression; +import org.apache.iotdb.commons.schema.table.TreeViewSchema; +import org.apache.iotdb.commons.schema.table.TsTable; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; +import org.apache.iotdb.commons.subscription.columnfilter.ColumnMetadata; +import org.apache.iotdb.rpc.subscription.config.TopicConfig; +import org.apache.iotdb.rpc.subscription.config.TopicConstant; +import org.apache.iotdb.rpc.subscription.exception.SubscriptionException; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Objects; +import java.util.Set; + +public class ColumnFilterBinder { + + private final ColumnFilterParser parser = new ColumnFilterParser(); + + public BoundColumnFilter bind( + final TopicConfig topicConfig, final Map> tables) + throws SubscriptionException { + if (Objects.isNull(topicConfig) + || !topicConfig.isTableTopic() + || topicConfig.isColumnFilterTrivial()) { + return BoundColumnFilter.matchAll(); + } + + final Expression expression = parser.parseAndValidate(topicConfig.getColumnFilter()); + final TablePattern tablePattern = buildTablePattern(topicConfig); + final Map> selectedColumnsByTable = new HashMap<>(); + final Map timeSelectedByTable = new HashMap<>(); + boolean timeSelected = false; + + for (final Map.Entry> databaseEntry : tables.entrySet()) { + final String database = databaseEntry.getKey(); + if (!tablePattern.matchesDatabase(database)) { + continue; + } + for (final TsTable table : databaseEntry.getValue().values()) { + if (Objects.isNull(table) || !tablePattern.matchesTable(table.getTableName())) { + continue; + } + final BindTableResult tableResult = bindTable(expression, database, table); + timeSelected |= tableResult.timeSelected; + timeSelectedByTable.put( + BoundColumnFilter.TableKey.of(database, table.getTableName()), + tableResult.timeSelected); + if (!tableResult.selectedColumnNames.isEmpty()) { + selectedColumnsByTable.put( + BoundColumnFilter.TableKey.of(database, table.getTableName()), + tableResult.selectedColumnNames); + } + } + } + + return BoundColumnFilter.of(selectedColumnsByTable, timeSelected, timeSelectedByTable); + } + + private static BindTableResult bindTable( + final Expression expression, final String database, final TsTable table) { + boolean hasMatchedColumn = false; + boolean timeSelected = false; + final Set selectedColumnNames = new HashSet<>(); + final Set tagColumnNames = new HashSet<>(); + + for (final TsTableColumnSchema columnSchema : table.getColumnList()) { + if (Objects.isNull(columnSchema)) { + continue; + } + if (columnSchema.getColumnCategory() == TsTableColumnCategory.TAG) { + tagColumnNames.add(BoundColumnFilter.normalize(sourceColumnName(table, columnSchema))); + } + if (!evaluate(expression, database, table.getTableName(), columnSchema)) { + continue; + } + + hasMatchedColumn = true; + switch (columnSchema.getColumnCategory()) { + case TIME: + timeSelected = true; + break; + case ATTRIBUTE: + break; + case TAG: + case FIELD: + selectedColumnNames.add( + BoundColumnFilter.normalize(sourceColumnName(table, columnSchema))); + break; + default: + throw new IllegalArgumentException( + "Unsupported table column category: " + columnSchema.getColumnCategory()); + } + } + + if (hasMatchedColumn) { + selectedColumnNames.addAll(tagColumnNames); + } + return new BindTableResult(selectedColumnNames, timeSelected); + } + + private static boolean evaluate( + final Expression expression, + final String database, + final String tableName, + final TsTableColumnSchema columnSchema) { + return ColumnFilterEvaluator.evaluate( + expression, + new ColumnMetadata( + database, + tableName, + columnSchema.getColumnName(), + columnSchema.getDataType().name(), + columnSchema.getColumnCategory().name())); + } + + private static String sourceColumnName( + final TsTable table, final TsTableColumnSchema columnSchema) { + return TreeViewSchema.isTreeViewTable(table) + && columnSchema.getColumnCategory() == TsTableColumnCategory.FIELD + ? TreeViewSchema.getSourceName(columnSchema) + : columnSchema.getColumnName(); + } + + private static TablePattern buildTablePattern(final TopicConfig topicConfig) { + return new TablePattern( + true, + topicConfig.getStringOrDefault( + TopicConstant.DATABASE_KEY, TopicConstant.DATABASE_DEFAULT_VALUE), + topicConfig.getStringOrDefault(TopicConstant.TABLE_KEY, TopicConstant.TABLE_DEFAULT_VALUE)); + } + + private static final class BindTableResult { + + private final Set selectedColumnNames; + private final boolean timeSelected; + + private BindTableResult(final Set selectedColumnNames, final boolean timeSelected) { + this.selectedColumnNames = selectedColumnNames; + this.timeSelected = timeSelected; + } + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/columnfilter/ColumnFilterEvaluator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/columnfilter/ColumnFilterEvaluator.java new file mode 100644 index 0000000000000..eb52e63c0d3c5 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/columnfilter/ColumnFilterEvaluator.java @@ -0,0 +1,179 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.subscription.columnfilter; + +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.BooleanLiteral; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.CommonQueryAstVisitor; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.ComparisonExpression; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.Expression; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.FunctionCall; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.Identifier; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.InListExpression; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.InPredicate; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.IsNullPredicate; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.LikePredicate; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.LogicalExpression; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.Node; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.NotExpression; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.StringLiteral; +import org.apache.iotdb.commons.subscription.columnfilter.ColumnMetadata; + +import java.util.Locale; +import java.util.Objects; +import java.util.regex.Pattern; + +public class ColumnFilterEvaluator implements CommonQueryAstVisitor { + + public static boolean evaluate(final Expression expression, final ColumnMetadata metadata) { + return Boolean.TRUE.equals(new ColumnFilterEvaluator().process(expression, metadata)); + } + + @Override + public Boolean visitNode(final Node node, final ColumnMetadata context) { + throw new IllegalArgumentException( + "unsupported expression: " + node.getClass().getSimpleName()); + } + + @Override + public Boolean visitBooleanLiteral(final BooleanLiteral node, final ColumnMetadata context) { + return node.getValue(); + } + + @Override + public Boolean visitLogicalExpression( + final LogicalExpression node, final ColumnMetadata context) { + if (node.getOperator() == LogicalExpression.Operator.AND) { + for (final Expression term : node.getTerms()) { + if (!process(term, context)) { + return false; + } + } + return true; + } + + for (final Expression term : node.getTerms()) { + if (process(term, context)) { + return true; + } + } + return false; + } + + @Override + public Boolean visitNotExpression(final NotExpression node, final ColumnMetadata context) { + return !process(node.getValue(), context); + } + + @Override + public Boolean visitComparisonExpression( + final ComparisonExpression node, final ColumnMetadata context) { + final String left = fieldValue((Identifier) node.getLeft(), context); + final String right = ((StringLiteral) node.getRight()).getValue(); + final boolean equals = equalsIgnoreCase(left, right); + return node.getOperator() == ComparisonExpression.Operator.EQUAL ? equals : !equals; + } + + @Override + public Boolean visitInPredicate(final InPredicate node, final ColumnMetadata context) { + final String left = fieldValue((Identifier) node.getValue(), context); + for (final Expression expression : ((InListExpression) node.getValueList()).getValues()) { + if (equalsIgnoreCase(left, ((StringLiteral) expression).getValue())) { + return true; + } + } + return false; + } + + @Override + public Boolean visitLikePredicate(final LikePredicate node, final ColumnMetadata context) { + final String left = fieldValue((Identifier) node.getValue(), context); + final String pattern = ((StringLiteral) node.getPattern()).getValue(); + final String escape = + node.getEscape().map(expression -> ((StringLiteral) expression).getValue()).orElse(null); + return compileLikePattern(pattern, escape).matcher(left).matches(); + } + + @Override + public Boolean visitFunctionCall(final FunctionCall node, final ColumnMetadata context) { + final String left = fieldValue((Identifier) node.getArguments().get(0), context); + final String pattern = ((StringLiteral) node.getArguments().get(1)).getValue(); + return Pattern.compile(pattern, Pattern.CASE_INSENSITIVE).matcher(left).matches(); + } + + @Override + public Boolean visitIsNullPredicate(final IsNullPredicate node, final ColumnMetadata context) { + return Objects.isNull(fieldValue((Identifier) node.getValue(), context)); + } + + static Pattern compileLikePattern(final String pattern, final String escape) { + final Character escapeChar; + if (Objects.isNull(escape)) { + escapeChar = null; + } else if (escape.length() == 1) { + escapeChar = escape.charAt(0); + } else { + throw new IllegalArgumentException("LIKE escape must be a single character"); + } + + final StringBuilder regex = new StringBuilder(); + boolean escaping = false; + for (int i = 0; i < pattern.length(); i++) { + final char ch = pattern.charAt(i); + if (Objects.nonNull(escapeChar) && ch == escapeChar && !escaping) { + escaping = true; + continue; + } + if (!escaping && ch == '%') { + regex.append(".*"); + } else if (!escaping && ch == '_') { + regex.append('.'); + } else { + regex.append(Pattern.quote(String.valueOf(ch))); + } + escaping = false; + } + if (escaping) { + throw new IllegalArgumentException("LIKE pattern ends with escape character"); + } + return Pattern.compile(regex.toString(), Pattern.CASE_INSENSITIVE); + } + + private static String fieldValue(final Identifier field, final ColumnMetadata metadata) { + switch (ColumnFilterValidator.normalizeField(field.getValue())) { + case "database": + return metadata.getDatabase(); + case "table_name": + return metadata.getTableName(); + case "column_name": + return metadata.getColumnName(); + case "datatype": + return metadata.getDatatype(); + case "category": + return metadata.getCategory(); + default: + throw new IllegalArgumentException( + "unsupported column metadata field: " + field.getValue()); + } + } + + private static boolean equalsIgnoreCase(final String left, final String right) { + return left.toLowerCase(Locale.ROOT).equals(right.toLowerCase(Locale.ROOT)); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/columnfilter/ColumnFilterMatcher.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/columnfilter/ColumnFilterMatcher.java new file mode 100644 index 0000000000000..4aa6fe51ccc61 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/columnfilter/ColumnFilterMatcher.java @@ -0,0 +1,204 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.subscription.columnfilter; + +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.Expression; +import org.apache.iotdb.commons.subscription.columnfilter.ColumnMetadata; +import org.apache.iotdb.rpc.subscription.config.TopicConfig; +import org.apache.iotdb.rpc.subscription.exception.SubscriptionException; + +import org.apache.tsfile.enums.ColumnCategory; +import org.apache.tsfile.enums.TSDataType; + +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; +import java.util.Set; + +public class ColumnFilterMatcher { + + private static final ColumnFilterMatcher MATCH_ALL = new ColumnFilterMatcher(null, null); + + private final BoundColumnFilter boundColumnFilter; + private final Set selectedColumnNames; + private final Expression expression; + + private ColumnFilterMatcher(final Set selectedColumnNames, final Expression expression) { + this(null, selectedColumnNames, expression); + } + + private ColumnFilterMatcher( + final BoundColumnFilter boundColumnFilter, + final Set selectedColumnNames, + final Expression expression) { + this.boundColumnFilter = boundColumnFilter; + this.selectedColumnNames = selectedColumnNames; + this.expression = expression; + } + + public static ColumnFilterMatcher matchAll() { + return MATCH_ALL; + } + + public static ColumnFilterMatcher fromBoundColumnFilter( + final BoundColumnFilter boundColumnFilter) { + if (Objects.isNull(boundColumnFilter) || boundColumnFilter.isMatchAll()) { + return matchAll(); + } + return new ColumnFilterMatcher(boundColumnFilter, null, null); + } + + public static ColumnFilterMatcher fromTopicConfig(final TopicConfig topicConfig) + throws SubscriptionException { + if (Objects.isNull(topicConfig) + || !topicConfig.isTableTopic() + || topicConfig.isColumnFilterTrivial()) { + return matchAll(); + } + return new ColumnFilterMatcher( + null, new ColumnFilterParser().parseAndValidate(topicConfig.getColumnFilter())); + } + + public static ColumnFilterMatcher ofSelectedColumnNames(final Set selectedColumnNames) { + if (Objects.isNull(selectedColumnNames)) { + return matchAll(); + } + + final Set normalizedColumnNames = new HashSet<>(); + selectedColumnNames.stream() + .filter(Objects::nonNull) + .map(ColumnFilterMatcher::normalize) + .forEach(normalizedColumnNames::add); + return new ColumnFilterMatcher(Collections.unmodifiableSet(normalizedColumnNames), null); + } + + public boolean isMatchAll() { + return Objects.isNull(boundColumnFilter) + && Objects.isNull(selectedColumnNames) + && Objects.isNull(expression); + } + + public boolean shouldAutoRetainTagsAtRuntime() { + return Objects.isNull(boundColumnFilter); + } + + public boolean isTimeSelected() { + if (isMatchAll()) { + return true; + } + if (Objects.nonNull(boundColumnFilter)) { + return boundColumnFilter.isTimeSelected(); + } + if (Objects.nonNull(selectedColumnNames)) { + return selectedColumnNames.contains("time"); + } + return isTimeSelected("", ""); + } + + public boolean isTimeSelected(final String databaseName, final String tableName) { + if (isMatchAll()) { + return true; + } + if (Objects.nonNull(boundColumnFilter)) { + return boundColumnFilter.isTimeSelected(databaseName, tableName); + } + if (Objects.nonNull(selectedColumnNames)) { + return selectedColumnNames.contains("time"); + } + return ColumnFilterEvaluator.evaluate( + expression, + new ColumnMetadata( + normalizeNullable(databaseName), + normalizeNullable(tableName), + "time", + TSDataType.TIMESTAMP.name(), + ColumnCategory.TIME.name())); + } + + public Map> getTimeSelectedByTable(final String databaseName) { + if (Objects.isNull(boundColumnFilter) || boundColumnFilter.getTimeSelectedByTable().isEmpty()) { + return Collections.emptyMap(); + } + + final String normalizedDatabaseName = + Objects.nonNull(databaseName) ? normalize(databaseName) : null; + final Map> result = new HashMap<>(); + boundColumnFilter + .getTimeSelectedByTable() + .forEach( + (tableKey, timeSelected) -> { + if (Objects.nonNull(normalizedDatabaseName) + && !Objects.equals(normalizedDatabaseName, tableKey.getDatabaseName())) { + return; + } + result + .computeIfAbsent(tableKey.getDatabaseName(), ignored -> new HashMap<>()) + .put(tableKey.getTableName(), timeSelected); + }); + if (result.isEmpty()) { + return Collections.emptyMap(); + } + + final Map> copied = new HashMap<>(); + result.forEach( + (database, tableMap) -> copied.put(database, Collections.unmodifiableMap(tableMap))); + return Collections.unmodifiableMap(copied); + } + + public boolean match(final String databaseName, final String tableName, final String columnName) { + return match(databaseName, tableName, columnName, null, null); + } + + public boolean match( + final String databaseName, + final String tableName, + final String columnName, + final TSDataType dataType, + final ColumnCategory category) { + if (isMatchAll()) { + return true; + } + if (Objects.nonNull(boundColumnFilter)) { + return boundColumnFilter.match(databaseName, tableName, columnName); + } + if (Objects.nonNull(selectedColumnNames)) { + return selectedColumnNames.contains(normalize(columnName)); + } + return ColumnFilterEvaluator.evaluate( + expression, + new ColumnMetadata( + normalizeNullable(databaseName), + normalizeNullable(tableName), + normalizeNullable(columnName), + Objects.nonNull(dataType) ? dataType.name() : "", + Objects.nonNull(category) ? category.name() : "")); + } + + private static String normalize(final String value) { + return value.trim().toLowerCase(Locale.ROOT); + } + + private static String normalizeNullable(final String value) { + return Objects.nonNull(value) ? value.trim() : ""; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/columnfilter/ColumnFilterParser.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/columnfilter/ColumnFilterParser.java new file mode 100644 index 0000000000000..8b785988b2587 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/columnfilter/ColumnFilterParser.java @@ -0,0 +1,274 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.subscription.columnfilter; + +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.BooleanLiteral; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.ComparisonExpression; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.Expression; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.FunctionCall; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.Identifier; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.InListExpression; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.InPredicate; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.IsNullPredicate; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.LikePredicate; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.LogicalExpression; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.NotExpression; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.QualifiedName; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.StringLiteral; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.parser.ParsingException; +import org.apache.iotdb.db.relational.grammar.sql.ColumnFilterBaseVisitor; +import org.apache.iotdb.db.relational.grammar.sql.ColumnFilterLexer; +import org.apache.iotdb.rpc.subscription.exception.SubscriptionException; + +import org.antlr.v4.runtime.BaseErrorListener; +import org.antlr.v4.runtime.CharStreams; +import org.antlr.v4.runtime.CommonTokenStream; +import org.antlr.v4.runtime.DefaultErrorStrategy; +import org.antlr.v4.runtime.InputMismatchException; +import org.antlr.v4.runtime.Parser; +import org.antlr.v4.runtime.RecognitionException; +import org.antlr.v4.runtime.Recognizer; +import org.antlr.v4.runtime.Token; +import org.antlr.v4.runtime.atn.PredictionMode; +import org.antlr.v4.runtime.tree.TerminalNode; + +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.regex.Pattern; + +public class ColumnFilterParser { + + private static final Pattern SINGLE_FIELD_PATTERN = + Pattern.compile("\\s*(?:[A-Za-z_][A-Za-z_0-9]*|\"(?:\"\"|[^\"])*\")\\s*"); + private static final Pattern FUNCTION_CALL_START_PATTERN = + Pattern.compile("\\s*(?:[A-Za-z_][A-Za-z_0-9]*|\"(?:\"\"|[^\"])*\")\\s*\\(.*"); + private static final Pattern UNQUOTED_COMPARISON_RIGHT_PATTERN = + Pattern.compile("(?is).*(?:!=|<>|=)\\s*[A-Za-z_][A-Za-z_0-9]*\\s*"); + + private static final BaseErrorListener ERROR_LISTENER = + new BaseErrorListener() { + @Override + public void syntaxError( + final Recognizer recognizer, + final Object offendingSymbol, + final int line, + final int charPositionInLine, + final String message, + final RecognitionException e) { + throw new ParsingException(message, e, line, charPositionInLine + 1); + } + }; + + public Expression parseAndValidate(final String rawColumnFilter) throws SubscriptionException { + try { + final Expression expression = parse(rawColumnFilter); + ColumnFilterValidator.validate(expression); + return expression; + } catch (final ParsingException | IllegalArgumentException e) { + throw new SubscriptionException( + String.format("Invalid column-filter: %s", e.getMessage()), e); + } + } + + Expression parse(final String rawColumnFilter) { + if (rawColumnFilter == null || rawColumnFilter.trim().isEmpty()) { + throw new ParsingException("column-filter should not be empty", null, 1, 1); + } + validateUnsupportedSyntax(rawColumnFilter); + + final ColumnFilterLexer lexer = new ColumnFilterLexer(CharStreams.fromString(rawColumnFilter)); + final CommonTokenStream tokenStream = new CommonTokenStream(lexer); + final org.apache.iotdb.db.relational.grammar.sql.ColumnFilterParser parser = + new org.apache.iotdb.db.relational.grammar.sql.ColumnFilterParser(tokenStream); + + lexer.removeErrorListeners(); + lexer.addErrorListener(ERROR_LISTENER); + parser.removeErrorListeners(); + parser.addErrorListener(ERROR_LISTENER); + parser.setErrorHandler( + new DefaultErrorStrategy() { + @Override + public Token recoverInline(final Parser recognizer) throws RecognitionException { + if (nextTokensContext == null) { + throw new InputMismatchException(recognizer); + } + throw new InputMismatchException(recognizer, nextTokensState, nextTokensContext); + } + }); + + try { + parser.getInterpreter().setPredictionMode(PredictionMode.SLL); + return new AstBuilder().visit(parser.columnFilter()); + } catch (final ParsingException e) { + tokenStream.seek(0); + parser.reset(); + parser.getInterpreter().setPredictionMode(PredictionMode.LL); + return new AstBuilder().visit(parser.columnFilter()); + } + } + + private static void validateUnsupportedSyntax(final String rawColumnFilter) { + final String trimmedColumnFilter = rawColumnFilter.trim(); + if ((SINGLE_FIELD_PATTERN.matcher(rawColumnFilter).matches() + && !"true".equalsIgnoreCase(trimmedColumnFilter) + && !"false".equalsIgnoreCase(trimmedColumnFilter)) + || FUNCTION_CALL_START_PATTERN.matcher(rawColumnFilter).matches()) { + throw new ParsingException("expected column predicate operator", null, 1, 1); + } + if (UNQUOTED_COMPARISON_RIGHT_PATTERN.matcher(rawColumnFilter).matches()) { + throw new ParsingException("expected string literal", null, 1, 1); + } + for (int i = 0; i < rawColumnFilter.length(); i++) { + final char ch = rawColumnFilter.charAt(i); + if (ch == '<') { + if (i + 1 < rawColumnFilter.length() && rawColumnFilter.charAt(i + 1) == '>') { + i++; + continue; + } + throw new ParsingException("unsupported comparison operator '<'", null, 1, i + 1); + } + if (ch == '>') { + throw new ParsingException("unsupported comparison operator '>'", null, 1, i + 1); + } + if (ch == '+') { + throw new ParsingException("unexpected character '+'", null, 1, i + 1); + } + } + } + + private static class AstBuilder extends ColumnFilterBaseVisitor { + + @Override + public Expression visitColumnFilter( + final org.apache.iotdb.db.relational.grammar.sql.ColumnFilterParser.ColumnFilterContext + context) { + return visit(context.booleanExpression()); + } + + @Override + public Expression visitPredicateExpression( + final org.apache.iotdb.db.relational.grammar.sql.ColumnFilterParser + .PredicateExpressionContext + context) { + return visit(context.predicate()); + } + + @Override + public Expression visitLogicalNot( + final org.apache.iotdb.db.relational.grammar.sql.ColumnFilterParser.LogicalNotContext + context) { + return new NotExpression(visit(context.booleanExpression())); + } + + @Override + public Expression visitLogicalBinary( + final org.apache.iotdb.db.relational.grammar.sql.ColumnFilterParser.LogicalBinaryContext + context) { + final Expression left = visit(context.booleanExpression(0)); + final Expression right = visit(context.booleanExpression(1)); + return Objects.nonNull(context.AND()) + ? LogicalExpression.and(left, right) + : LogicalExpression.or(left, right); + } + + @Override + public Expression visitPredicate( + final org.apache.iotdb.db.relational.grammar.sql.ColumnFilterParser.PredicateContext + context) { + if (Objects.nonNull(context.booleanValue())) { + return visit(context.booleanValue()); + } + if (Objects.nonNull(context.booleanExpression())) { + return visit(context.booleanExpression()); + } + + final Identifier field = toIdentifier(context.field()); + if (Objects.nonNull(context.comparisonOperator())) { + return new ComparisonExpression( + Objects.nonNull(context.comparisonOperator().EQ()) + ? ComparisonExpression.Operator.EQUAL + : ComparisonExpression.Operator.NOT_EQUAL, + field, + toStringLiteral(context.string(0))); + } + if (Objects.nonNull(context.IN())) { + final List values = new ArrayList<>(); + for (final org.apache.iotdb.db.relational.grammar.sql.ColumnFilterParser.StringContext + string : context.string()) { + values.add(toStringLiteral(string)); + } + return maybeNegate( + new InPredicate(field, new InListExpression(values)), Objects.nonNull(context.NOT())); + } + if (Objects.nonNull(context.LIKE())) { + final Expression like = + context.string().size() > 1 + ? new LikePredicate( + field, toStringLiteral(context.string(0)), toStringLiteral(context.string(1))) + : new LikePredicate(field, toStringLiteral(context.string(0))); + return maybeNegate(like, Objects.nonNull(context.NOT())); + } + if (Objects.nonNull(context.REGEXP())) { + final Expression regexp = + new FunctionCall( + QualifiedName.of("regexp_like"), + List.of(field, toStringLiteral(context.string(0)))); + return maybeNegate(regexp, Objects.nonNull(context.NOT())); + } + if (Objects.nonNull(context.IS())) { + return maybeNegate(new IsNullPredicate(field), Objects.nonNull(context.NOT())); + } + + throw new IllegalArgumentException("unsupported column-filter predicate"); + } + + @Override + public Expression visitBooleanValue( + final org.apache.iotdb.db.relational.grammar.sql.ColumnFilterParser.BooleanValueContext + context) { + return Objects.nonNull(context.TRUE()) + ? BooleanLiteral.TRUE_LITERAL + : BooleanLiteral.FALSE_LITERAL; + } + + private static Expression maybeNegate(final Expression expression, final boolean negated) { + return negated ? new NotExpression(expression) : expression; + } + + private static Identifier toIdentifier( + final org.apache.iotdb.db.relational.grammar.sql.ColumnFilterParser.FieldContext context) { + final TerminalNode quoted = context.QUOTED_IDENTIFIER(); + if (Objects.nonNull(quoted)) { + return new Identifier(unquote(quoted.getText()), true); + } + return new Identifier(context.IDENTIFIER().getText()); + } + + private static StringLiteral toStringLiteral( + final org.apache.iotdb.db.relational.grammar.sql.ColumnFilterParser.StringContext context) { + return new StringLiteral(unquote(context.QUOTED_IDENTIFIER().getText())); + } + + private static String unquote(final String text) { + return text.substring(1, text.length() - 1).replace("\"\"", "\""); + } + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/columnfilter/ColumnFilterValidator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/columnfilter/ColumnFilterValidator.java new file mode 100644 index 0000000000000..d527e7eeadb1c --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/columnfilter/ColumnFilterValidator.java @@ -0,0 +1,162 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.subscription.columnfilter; + +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.BooleanLiteral; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.CommonQueryAstVisitor; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.ComparisonExpression; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.Expression; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.FunctionCall; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.Identifier; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.InListExpression; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.InPredicate; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.IsNullPredicate; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.LikePredicate; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.LogicalExpression; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.Node; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.NotExpression; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.StringLiteral; + +import java.util.Locale; +import java.util.Set; +import java.util.regex.Pattern; +import java.util.regex.PatternSyntaxException; + +public class ColumnFilterValidator implements CommonQueryAstVisitor { + + private static final Set LEGAL_FIELDS = + Set.of("database", "table_name", "column_name", "datatype", "category"); + private static final String REGEXP_LIKE = "regexp_like"; + + public static void validate(final Expression expression) { + new ColumnFilterValidator().process(expression); + } + + @Override + public Void visitNode(final Node node, final Void context) { + throw invalid("unsupported expression: " + node.getClass().getSimpleName()); + } + + @Override + public Void visitBooleanLiteral(final BooleanLiteral node, final Void context) { + return null; + } + + @Override + public Void visitLogicalExpression(final LogicalExpression node, final Void context) { + node.getTerms().forEach(this::process); + return null; + } + + @Override + public Void visitNotExpression(final NotExpression node, final Void context) { + process(node.getValue()); + return null; + } + + @Override + public Void visitComparisonExpression(final ComparisonExpression node, final Void context) { + if (node.getOperator() != ComparisonExpression.Operator.EQUAL + && node.getOperator() != ComparisonExpression.Operator.NOT_EQUAL) { + throw invalid("only =, !=, and <> comparisons are supported in column-filter"); + } + requireField(node.getLeft()); + requireStringLiteral(node.getRight(), "comparison right operand"); + return null; + } + + @Override + public Void visitInPredicate(final InPredicate node, final Void context) { + requireField(node.getValue()); + if (!(node.getValueList() instanceof InListExpression)) { + throw invalid("IN predicate must use a string literal list"); + } + for (final Expression expression : ((InListExpression) node.getValueList()).getValues()) { + requireStringLiteral(expression, "IN element"); + } + return null; + } + + @Override + public Void visitLikePredicate(final LikePredicate node, final Void context) { + requireField(node.getValue()); + final StringLiteral pattern = requireStringLiteral(node.getPattern(), "LIKE pattern"); + final String escape = + node.getEscape() + .map(expression -> requireStringLiteral(expression, "LIKE escape").getValue()) + .orElse(null); + ColumnFilterEvaluator.compileLikePattern(pattern.getValue(), escape); + return null; + } + + @Override + public Void visitFunctionCall(final FunctionCall node, final Void context) { + if (!REGEXP_LIKE.equalsIgnoreCase(node.getName().toString()) + || node.isDistinct() + || node.getProcessingMode().isPresent() + || node.getArguments().size() != 2) { + throw invalid("only REGEXP is supported as regexp_like(field, pattern)"); + } + + requireField(node.getArguments().get(0)); + final String pattern = + requireStringLiteral(node.getArguments().get(1), "REGEXP pattern").getValue(); + try { + Pattern.compile(pattern, Pattern.CASE_INSENSITIVE); + } catch (final PatternSyntaxException e) { + throw invalid("illegal REGEXP pattern: " + e.getMessage()); + } + return null; + } + + @Override + public Void visitIsNullPredicate(final IsNullPredicate node, final Void context) { + requireField(node.getValue()); + return null; + } + + private static Identifier requireField(final Expression expression) { + if (!(expression instanceof Identifier)) { + throw invalid("left operand must be one of column metadata fields"); + } + final Identifier identifier = (Identifier) expression; + final String normalizedField = normalizeField(identifier.getValue()); + if (!LEGAL_FIELDS.contains(normalizedField)) { + throw invalid("unsupported column metadata field: " + identifier.getValue()); + } + return identifier; + } + + private static StringLiteral requireStringLiteral( + final Expression expression, final String description) { + if (!(expression instanceof StringLiteral)) { + throw invalid(description + " must be a string literal"); + } + return (StringLiteral) expression; + } + + static String normalizeField(final String fieldName) { + return fieldName.trim().toLowerCase(Locale.ROOT); + } + + private static IllegalArgumentException invalid(final String message) { + return new IllegalArgumentException(message); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/columnfilter/TabletColumnPruner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/columnfilter/TabletColumnPruner.java new file mode 100644 index 0000000000000..2d7ebd240f2e7 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/columnfilter/TabletColumnPruner.java @@ -0,0 +1,173 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.subscription.columnfilter; + +import org.apache.tsfile.enums.ColumnCategory; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.utils.BitMap; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Objects; + +/** Prunes table-model Tablets according to a subscription column matcher. */ +public class TabletColumnPruner { + + private TabletColumnPruner() { + // utility class + } + + public static Tablet pruneTableModelTablet( + final Tablet tablet, final String databaseName, final ColumnFilterMatcher matcher) { + if (Objects.isNull(tablet) || Objects.isNull(databaseName)) { + return tablet; + } + + final ColumnFilterMatcher effectiveMatcher = + Objects.nonNull(matcher) ? matcher : ColumnFilterMatcher.matchAll(); + final List schemas = tablet.getSchemas(); + final Object[] values = tablet.getValues(); + if (Objects.isNull(schemas) || schemas.isEmpty() || Objects.isNull(values)) { + return null; + } + + final List categories = getColumnCategories(tablet, schemas.size()); + final boolean[] selectedColumns = new boolean[schemas.size()]; + boolean hasMatchedColumn = false; + + for (int i = 0; i < schemas.size(); i++) { + if (!isValidColumn(schemas, values, i)) { + continue; + } + final IMeasurementSchema schema = schemas.get(i); + final ColumnCategory category = categories.get(i); + if (effectiveMatcher.match( + databaseName, + tablet.getTableName(), + schema.getMeasurementName(), + schema.getType(), + category)) { + hasMatchedColumn = true; + if (category != ColumnCategory.ATTRIBUTE) { + selectedColumns[i] = true; + } + } + } + + if (!hasMatchedColumn) { + return null; + } + + if (effectiveMatcher.shouldAutoRetainTagsAtRuntime()) { + for (int i = 0; i < schemas.size(); i++) { + if (isValidColumn(schemas, values, i) + && categories.get(i) == ColumnCategory.TAG + && hasValueArray(values, i)) { + selectedColumns[i] = true; + } + } + } + + final List selectedIndices = getSelectedIndices(selectedColumns); + if (selectedIndices.isEmpty()) { + return null; + } + if (selectedIndices.size() == schemas.size()) { + return tablet; + } + + final List prunedSchemas = new ArrayList<>(selectedIndices.size()); + final List prunedCategories = new ArrayList<>(selectedIndices.size()); + final Object[] prunedValues = new Object[selectedIndices.size()]; + final BitMap[] bitMaps = tablet.getBitMaps(); + final BitMap[] prunedBitMaps = + Objects.nonNull(bitMaps) ? new BitMap[selectedIndices.size()] : null; + + for (int i = 0; i < selectedIndices.size(); i++) { + final int originalIndex = selectedIndices.get(i); + final IMeasurementSchema originalSchema = schemas.get(originalIndex); + prunedSchemas.add( + new MeasurementSchema(originalSchema.getMeasurementName(), originalSchema.getType())); + prunedCategories.add(categories.get(originalIndex)); + prunedValues[i] = values[originalIndex]; + if (Objects.nonNull(bitMaps) && originalIndex < bitMaps.length) { + prunedBitMaps[i] = bitMaps[originalIndex]; + } + } + + return new Tablet( + tablet.getTableName(), + prunedSchemas, + prunedCategories, + tablet.getTimestamps(), + prunedValues, + prunedBitMaps, + tablet.getRowSize()); + } + + private static List getColumnCategories( + final Tablet tablet, final int columnCount) { + final List categories = tablet.getColumnTypes(); + if (Objects.isNull(categories) || categories.isEmpty()) { + return Collections.nCopies(columnCount, ColumnCategory.FIELD); + } + final List result = new ArrayList<>(columnCount); + for (int i = 0; i < columnCount; i++) { + result.add( + i < categories.size() && Objects.nonNull(categories.get(i)) + ? categories.get(i) + : ColumnCategory.FIELD); + } + return result; + } + + private static boolean isValidColumn( + final List schemas, final Object[] values, final int index) { + if (index < 0 || index >= schemas.size() || index >= values.length) { + return false; + } + final IMeasurementSchema schema = schemas.get(index); + final TSDataType dataType = Objects.nonNull(schema) ? schema.getType() : null; + return Objects.nonNull(schema) + && Objects.nonNull(schema.getMeasurementName()) + && Objects.nonNull(dataType); + } + + private static boolean hasValueArray(final Object[] values, final int index) { + return Objects.nonNull(values) + && index >= 0 + && index < values.length + && Objects.nonNull(values[index]); + } + + private static List getSelectedIndices(final boolean[] selectedColumns) { + final List result = new ArrayList<>(selectedColumns.length); + for (int i = 0; i < selectedColumns.length; i++) { + if (selectedColumns[i]) { + result.add(i); + } + } + return result; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/SubscriptionEvent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/SubscriptionEvent.java index facb00b219063..3c99a17f49fc2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/SubscriptionEvent.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/SubscriptionEvent.java @@ -43,6 +43,7 @@ import java.io.File; import java.io.IOException; import java.nio.ByteBuffer; +import java.util.Map; import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicLong; @@ -84,14 +85,38 @@ public SubscriptionEvent( final short responseType, final SubscriptionPollPayload payload, final SubscriptionCommitContext commitContext) { + this(responseType, payload, commitContext, true); + } + + public SubscriptionEvent( + final short responseType, + final SubscriptionPollPayload payload, + final SubscriptionCommitContext commitContext, + final boolean timeSelected) { + this(responseType, payload, commitContext, timeSelected, null); + } + + public SubscriptionEvent( + final short responseType, + final SubscriptionPollPayload payload, + final SubscriptionCommitContext commitContext, + final boolean timeSelected, + final Map> timeSelectedByTable) { this.pipeEvents = new SubscriptionPipeEmptyEvent(); - this.response = new SubscriptionEventSingleResponse(responseType, payload, commitContext); + this.response = + new SubscriptionEventSingleResponse( + responseType, payload, commitContext, timeSelected, timeSelectedByTable); this.commitContext = commitContext; } @TestOnly public SubscriptionEvent(final SubscriptionPollResponse response) { - this(response.getResponseType(), response.getPayload(), response.getCommitContext()); + this( + response.getResponseType(), + response.getPayload(), + response.getCommitContext(), + response.isTimeSelected(), + response.getTimeSelectedByTable()); } /** diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/batch/SubscriptionPipeTabletEventBatch.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/batch/SubscriptionPipeTabletEventBatch.java index d5843d1d99da2..c1e703a738ed0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/batch/SubscriptionPipeTabletEventBatch.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/batch/SubscriptionPipeTabletEventBatch.java @@ -27,6 +27,8 @@ import org.apache.iotdb.db.pipe.resource.memory.PipeMemoryWeightUtil; import org.apache.iotdb.db.subscription.agent.SubscriptionAgent; import org.apache.iotdb.db.subscription.broker.SubscriptionPrefetchingTabletQueue; +import org.apache.iotdb.db.subscription.columnfilter.ColumnFilterMatcher; +import org.apache.iotdb.db.subscription.columnfilter.TabletColumnPruner; import org.apache.iotdb.db.subscription.event.SubscriptionEvent; import org.apache.iotdb.metrics.core.utils.IoTDBMovingAverage; import org.apache.iotdb.pipe.api.event.dml.insertion.TabletInsertionEvent; @@ -39,6 +41,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.ArrayList; import java.util.Collections; import java.util.Iterator; import java.util.List; @@ -165,6 +168,7 @@ protected boolean shouldEmit() { private Pair> convertToTablets( final TabletInsertionEvent tabletInsertionEvent) { + final Pair> result; if (tabletInsertionEvent instanceof PipeInsertNodeTabletInsertionEvent) { final List tablets = ((PipeInsertNodeTabletInsertionEvent) tabletInsertionEvent).convertToTablets(); @@ -173,28 +177,51 @@ private Pair> convertToTablets( .map(PipeMemoryWeightUtil::calculateTabletSizeInBytes) .reduce(Long::sum) .orElse(0L)); - return new Pair<>( - ((PipeInsertNodeTabletInsertionEvent) tabletInsertionEvent).isTableModelEvent() - ? ((PipeInsertNodeTabletInsertionEvent) tabletInsertionEvent) - .getTableModelDatabaseName() - : null, - tablets); + result = + new Pair<>( + ((PipeInsertNodeTabletInsertionEvent) tabletInsertionEvent).isTableModelEvent() + ? ((PipeInsertNodeTabletInsertionEvent) tabletInsertionEvent) + .getTableModelDatabaseName() + : null, + tablets); } else if (tabletInsertionEvent instanceof PipeRawTabletInsertionEvent) { final Tablet tablet = ((PipeRawTabletInsertionEvent) tabletInsertionEvent).convertToTablet(); updateEstimatedRawTabletInsertionEventSize( PipeMemoryWeightUtil.calculateTabletSizeInBytes(tablet)); - return new Pair<>( - ((PipeRawTabletInsertionEvent) tabletInsertionEvent).isTableModelEvent() - ? ((PipeRawTabletInsertionEvent) tabletInsertionEvent).getTableModelDatabaseName() - : null, - Collections.singletonList(tablet)); + result = + new Pair<>( + ((PipeRawTabletInsertionEvent) tabletInsertionEvent).isTableModelEvent() + ? ((PipeRawTabletInsertionEvent) tabletInsertionEvent).getTableModelDatabaseName() + : null, + Collections.singletonList(tablet)); + } else { + LOGGER.warn( + "SubscriptionPipeTabletEventBatch {} only support convert PipeInsertNodeTabletInsertionEvent or PipeRawTabletInsertionEvent to tablet. Ignore {}.", + this, + tabletInsertionEvent); + return null; } - LOGGER.warn( - "SubscriptionPipeTabletEventBatch {} only support convert PipeInsertNodeTabletInsertionEvent or PipeRawTabletInsertionEvent to tablet. Ignore {}.", - this, - tabletInsertionEvent); - return null; + return pruneTablets(result); + } + + private Pair> pruneTablets(final Pair> tablets) { + if (Objects.isNull(tablets) || Objects.isNull(tablets.left) || Objects.isNull(tablets.right)) { + return tablets; + } + + final ColumnFilterMatcher matcher = + SubscriptionAgent.broker().getColumnFilterMatcher(prefetchingQueue.getTopicName()); + + final List prunedTablets = new ArrayList<>(tablets.right.size()); + for (final Tablet tablet : tablets.right) { + final Tablet prunedTablet = + TabletColumnPruner.pruneTableModelTablet(tablet, tablets.left, matcher); + if (Objects.nonNull(prunedTablet)) { + prunedTablets.add(prunedTablet); + } + } + return prunedTablets.isEmpty() ? null : new Pair<>(tablets.left, prunedTablets); } /////////////////////////////// estimator /////////////////////////////// diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/batch/SubscriptionPipeTsFileEventBatch.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/batch/SubscriptionPipeTsFileEventBatch.java index 089e25b063dd4..8ca733ca49fb4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/batch/SubscriptionPipeTsFileEventBatch.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/batch/SubscriptionPipeTsFileEventBatch.java @@ -21,7 +21,10 @@ import org.apache.iotdb.commons.pipe.event.EnrichedEvent; import org.apache.iotdb.db.pipe.sink.payload.evolvable.batch.PipeTabletEventTsFileBatch; +import org.apache.iotdb.db.subscription.agent.SubscriptionAgent; import org.apache.iotdb.db.subscription.broker.SubscriptionPrefetchingTsFileQueue; +import org.apache.iotdb.db.subscription.columnfilter.ColumnFilterMatcher; +import org.apache.iotdb.db.subscription.columnfilter.TabletColumnPruner; import org.apache.iotdb.db.subscription.event.SubscriptionEvent; import org.apache.iotdb.db.subscription.event.pipe.SubscriptionPipeTsFileBatchEvents; import org.apache.iotdb.pipe.api.event.dml.insertion.TabletInsertionEvent; @@ -29,11 +32,13 @@ import org.apache.iotdb.rpc.subscription.payload.poll.SubscriptionCommitContext; import org.apache.tsfile.utils.Pair; +import org.apache.tsfile.write.record.Tablet; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.concurrent.atomic.AtomicInteger; @@ -50,7 +55,9 @@ public SubscriptionPipeTsFileEventBatch( final int maxDelayInMs, final long maxBatchSizeInBytes) { super(regionId, prefetchingQueue, maxDelayInMs, maxBatchSizeInBytes); - this.batch = new PipeTabletEventTsFileBatch(maxDelayInMs, maxBatchSizeInBytes); + this.batch = + new PipeTabletEventTsFileBatch( + maxDelayInMs, maxBatchSizeInBytes, this::pruneTableModelTablet); } @Override @@ -91,11 +98,18 @@ protected void onTsFileInsertionEvent(final TsFileInsertionEvent event) { @Override protected List generateSubscriptionEvents() throws Exception { if (batch.isEmpty()) { - return null; + enrichedEvents.clear(); + return Collections.emptyList(); } final List events = new ArrayList<>(); final List> dbTsFilePairs = batch.sealTsFiles(); + if (dbTsFilePairs.isEmpty()) { + batch.decreaseEventsReferenceCount(this.getClass().getName(), true); + batch.onSuccess(); + enrichedEvents.clear(); + return Collections.emptyList(); + } final AtomicInteger ackReferenceCount = new AtomicInteger(dbTsFilePairs.size()); final AtomicInteger cleanReferenceCount = new AtomicInteger(dbTsFilePairs.size()); for (final Pair pair : dbTsFilePairs) { @@ -113,6 +127,12 @@ protected List generateSubscriptionEvents() throws Exception @Override protected boolean shouldEmit() { - return batch.shouldEmit(); + return (!enrichedEvents.isEmpty() && batch.isEmpty()) || batch.shouldEmit(); + } + + private Tablet pruneTableModelTablet(final String databaseName, final Tablet tablet) { + final ColumnFilterMatcher matcher = + SubscriptionAgent.broker().getColumnFilterMatcher(prefetchingQueue.getTopicName()); + return TabletColumnPruner.pruneTableModelTablet(tablet, databaseName, matcher); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/cache/CachedSubscriptionPollResponse.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/cache/CachedSubscriptionPollResponse.java index 1a0be808c62cd..63f1fb347b11a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/cache/CachedSubscriptionPollResponse.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/cache/CachedSubscriptionPollResponse.java @@ -42,8 +42,30 @@ public CachedSubscriptionPollResponse( super(responseType, payload, commitContext); } + public CachedSubscriptionPollResponse( + final short responseType, + final SubscriptionPollPayload payload, + final SubscriptionCommitContext commitContext, + final boolean timeSelected) { + super(responseType, payload, commitContext, timeSelected); + } + + public CachedSubscriptionPollResponse( + final short responseType, + final SubscriptionPollPayload payload, + final SubscriptionCommitContext commitContext, + final boolean timeSelected, + final Map> timeSelectedByTable) { + super(responseType, payload, commitContext, timeSelected, timeSelectedByTable); + } + public CachedSubscriptionPollResponse(final SubscriptionPollResponse response) { - super(response.getResponseType(), response.getPayload(), response.getCommitContext()); + super( + response.getResponseType(), + response.getPayload(), + response.getCommitContext(), + response.isTimeSelected(), + response.getTimeSelectedByTable()); } public void setMemoryBlock(final PipeFixedMemoryBlock memoryBlock) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/response/SubscriptionEventSingleResponse.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/response/SubscriptionEventSingleResponse.java index aca492125fb74..f9cb9db971c66 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/response/SubscriptionEventSingleResponse.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/response/SubscriptionEventSingleResponse.java @@ -50,6 +50,26 @@ public SubscriptionEventSingleResponse( this.response = new CachedSubscriptionPollResponse(responseType, payload, commitContext); } + public SubscriptionEventSingleResponse( + final short responseType, + final SubscriptionPollPayload payload, + final SubscriptionCommitContext commitContext, + final boolean timeSelected) { + this.response = + new CachedSubscriptionPollResponse(responseType, payload, commitContext, timeSelected); + } + + public SubscriptionEventSingleResponse( + final short responseType, + final SubscriptionPollPayload payload, + final SubscriptionCommitContext commitContext, + final boolean timeSelected, + final Map> timeSelectedByTable) { + this.response = + new CachedSubscriptionPollResponse( + responseType, payload, commitContext, timeSelected, timeSelectedByTable); + } + public SubscriptionEventSingleResponse(final SubscriptionPollResponse response) { this.response = new CachedSubscriptionPollResponse(response); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/response/SubscriptionEventTabletResponse.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/response/SubscriptionEventTabletResponse.java index a06267d7b7617..abf98dfbc8e0f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/response/SubscriptionEventTabletResponse.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/response/SubscriptionEventTabletResponse.java @@ -25,7 +25,9 @@ import org.apache.iotdb.db.pipe.resource.memory.PipeMemoryManager; import org.apache.iotdb.db.pipe.resource.memory.PipeMemoryWeightUtil; import org.apache.iotdb.db.pipe.resource.memory.PipeTabletMemoryBlock; +import org.apache.iotdb.db.subscription.agent.SubscriptionAgent; import org.apache.iotdb.db.subscription.broker.SubscriptionPrefetchingQueue; +import org.apache.iotdb.db.subscription.columnfilter.ColumnFilterMatcher; import org.apache.iotdb.db.subscription.event.SubscriptionEvent; import org.apache.iotdb.db.subscription.event.batch.SubscriptionPipeTabletEventBatch; import org.apache.iotdb.db.subscription.event.cache.CachedSubscriptionPollResponse; @@ -68,6 +70,7 @@ public class SubscriptionEventTabletResponse extends SubscriptionEventExtendable private final SubscriptionPipeTabletEventBatch batch; private final SubscriptionPrefetchingQueue queue; private final SubscriptionPipeTabletBatchEvents events; + private final ColumnFilterMatcher columnFilterMatcher; private final SubscriptionCommitContext commitContext; private final SubscriptionCommitContext rootCommitContext; @@ -91,6 +94,8 @@ public SubscriptionEventTabletResponse( this.commitContext = commitContext; this.rootCommitContext = rootCommitContext; + this.columnFilterMatcher = + SubscriptionAgent.broker().getColumnFilterMatcher(queue.getTopicName()); init(); } @@ -159,7 +164,8 @@ private synchronized CachedSubscriptionPollResponse generateEmptyTabletResponse( return new CachedSubscriptionPollResponse( SubscriptionPollResponseType.TABLETS.getType(), new TabletsPayload(Collections.emptyList(), nextOffset.incrementAndGet()), - commitContext); + commitContext, + isTimeSelected()); } private synchronized CachedSubscriptionPollResponse generateNextTabletResponse() @@ -173,7 +179,8 @@ private synchronized CachedSubscriptionPollResponse generateNextTabletResponse() return new CachedSubscriptionPollResponse( SubscriptionPollResponseType.TABLETS.getType(), new TabletsPayload(Collections.emptyList(), -totalTablets), - commitContext); + commitContext, + isTimeSelected()); } CachedSubscriptionPollResponse response = null; @@ -211,7 +218,9 @@ private synchronized CachedSubscriptionPollResponse generateNextTabletResponse() new CachedSubscriptionPollResponse( SubscriptionPollResponseType.TABLETS.getType(), new TabletsPayload(new HashMap<>(currentTablets), nextOffset.incrementAndGet()), - commitContext); + commitContext, + isTimeSelected(), + getTimeSelectedByTable(currentTablets)); break; } @@ -221,7 +230,9 @@ private synchronized CachedSubscriptionPollResponse generateNextTabletResponse() new CachedSubscriptionPollResponse( SubscriptionPollResponseType.TABLETS.getType(), new TabletsPayload(new HashMap<>(currentTablets), nextOffset.incrementAndGet()), - commitContext); + commitContext, + isTimeSelected(), + getTimeSelectedByTable(currentTablets)); break; } @@ -242,14 +253,17 @@ private synchronized CachedSubscriptionPollResponse generateNextTabletResponse() new CachedSubscriptionPollResponse( SubscriptionPollResponseType.TABLETS.getType(), new TabletsPayload(Collections.emptyList(), -totalTablets), - commitContext); + commitContext, + isTimeSelected()); hasNoMore = true; } else { response = new CachedSubscriptionPollResponse( SubscriptionPollResponseType.TABLETS.getType(), new TabletsPayload(new HashMap<>(currentTablets), nextOffset.incrementAndGet()), - commitContext); + commitContext, + isTimeSelected(), + getTimeSelectedByTable(currentTablets)); } } @@ -313,6 +327,34 @@ private void transportIterationSnapshot() { events.receiveIterationSnapshot(batch.sendIterationSnapshot()); } + private boolean isTimeSelected() { + return columnFilterMatcher.isTimeSelected(); + } + + private Map> getTimeSelectedByTable( + final Map> tablets) { + if (Objects.isNull(tablets) || tablets.isEmpty()) { + return Collections.emptyMap(); + } + final Map> result = new HashMap<>(); + tablets.forEach( + (databaseName, tabletList) -> { + if (Objects.isNull(databaseName) || Objects.isNull(tabletList)) { + return; + } + final Map tableMap = + result.computeIfAbsent(databaseName, ignored -> new HashMap<>()); + for (final Tablet tablet : tabletList) { + if (Objects.nonNull(tablet) && Objects.nonNull(tablet.getTableName())) { + tableMap.put( + tablet.getTableName(), + columnFilterMatcher.isTimeSelected(databaseName, tablet.getTableName())); + } + } + }); + return result; + } + /////////////////////////////// stringify /////////////////////////////// @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/response/SubscriptionEventTsFileResponse.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/response/SubscriptionEventTsFileResponse.java index ee8ef1fdb8952..06ea942874c75 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/response/SubscriptionEventTsFileResponse.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/response/SubscriptionEventTsFileResponse.java @@ -26,6 +26,7 @@ import org.apache.iotdb.db.pipe.resource.memory.PipeMemoryManager; import org.apache.iotdb.db.pipe.resource.memory.PipeTsFileMemoryBlock; import org.apache.iotdb.db.subscription.agent.SubscriptionAgent; +import org.apache.iotdb.db.subscription.columnfilter.ColumnFilterMatcher; import org.apache.iotdb.db.subscription.event.cache.CachedSubscriptionPollResponse; import org.apache.iotdb.rpc.subscription.exception.SubscriptionException; import org.apache.iotdb.rpc.subscription.payload.poll.FileInitPayload; @@ -64,6 +65,8 @@ public class SubscriptionEventTsFileResponse extends SubscriptionEventExtendable private final File tsFile; @Nullable private final String databaseName; private final SubscriptionCommitContext commitContext; + private final ColumnFilterMatcher columnFilterMatcher; + private final Map> timeSelectedByTable; public SubscriptionEventTsFileResponse( final File tsFile, @@ -74,6 +77,9 @@ public SubscriptionEventTsFileResponse( this.tsFile = tsFile; this.databaseName = databaseName; this.commitContext = commitContext; + this.columnFilterMatcher = + SubscriptionAgent.broker().getColumnFilterMatcher(commitContext.getTopicName()); + this.timeSelectedByTable = columnFilterMatcher.getTimeSelectedByTable(databaseName); init(); } @@ -123,7 +129,9 @@ private void init() { new CachedSubscriptionPollResponse( SubscriptionPollResponseType.FILE_INIT.getType(), new FileInitPayload(tsFile.getName()), - commitContext)); + commitContext, + isTimeSelected(), + timeSelectedByTable)); } private synchronized Optional generateNextTsFileResponse( @@ -174,7 +182,9 @@ private CachedSubscriptionPollResponse generateResponseWithPieceOrSealPayload( return new CachedSubscriptionPollResponse( SubscriptionPollResponseType.FILE_SEAL.getType(), new FileSealPayload(tsFile.getName(), tsFile.length(), databaseName), - commitContext); + commitContext, + isTimeSelected(), + timeSelectedByTable); } final long bufferSize; @@ -208,7 +218,9 @@ private CachedSubscriptionPollResponse generateResponseWithPieceOrSealPayload( new CachedSubscriptionPollResponse( SubscriptionPollResponseType.FILE_PIECE.getType(), new FilePiecePayload(tsFile.getName(), writingOffset + readLength, readBuffer), - commitContext); + commitContext, + isTimeSelected(), + timeSelectedByTable); // set fixed memory block for response response.setMemoryBlock(memoryBlock); @@ -260,6 +272,10 @@ private void waitForResourceEnough4Slicing(final long timeoutMs) throws Interrup "Wait for resource enough for slicing tsfile {} for {} seconds.", tsFile, waitTimeSeconds); } + private boolean isTimeSelected() { + return columnFilterMatcher.isTimeSelected(); + } + /////////////////////////////// stringify /////////////////////////////// @Override diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/batch/PipeTabletEventTsFileBatchTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/batch/PipeTabletEventTsFileBatchTest.java new file mode 100644 index 0000000000000..de0d85d5a4ced --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/batch/PipeTabletEventTsFileBatchTest.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.pipe.sink.payload.evolvable.batch; + +import org.apache.iotdb.db.pipe.event.common.tablet.PipeRawTabletInsertionEvent; + +import org.apache.tsfile.enums.ColumnCategory; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.write.record.Tablet; +import org.junit.Assert; +import org.junit.Test; + +import java.util.Arrays; + +public class PipeTabletEventTsFileBatchTest { + + @Test + public void testFullyPrunedTableModelTabletIsReleasedAndNotRetained() throws Exception { + final PipeTabletEventTsFileBatch batch = + new PipeTabletEventTsFileBatch(100_000, 1024 * 1024, (databaseName, tablet) -> null); + final PipeRawTabletInsertionEvent event = + new PipeRawTabletInsertionEvent( + true, "root.db", "db", "root.db", createTablet(), false, "pipe", 1L, null, null, false); + + Assert.assertFalse(batch.onEvent(event)); + Assert.assertTrue(batch.isEmpty()); + Assert.assertTrue(event.isReleased()); + Assert.assertEquals(0, event.getReferenceCount()); + + batch.close(); + } + + private static Tablet createTablet() { + final Tablet tablet = + new Tablet( + "sensors", + Arrays.asList("device", "temperature"), + Arrays.asList(TSDataType.STRING, TSDataType.DOUBLE), + Arrays.asList(ColumnCategory.TAG, ColumnCategory.FIELD), + 1); + tablet.addTimestamp(0, 1L); + tablet.addValue(0, 0, "d1"); + tablet.addValue(0, 1, 36.5); + tablet.setRowSize(1); + return tablet; + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/agent/SubscriptionTopicAgentTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/agent/SubscriptionTopicAgentTest.java new file mode 100644 index 0000000000000..c31535b5812fc --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/agent/SubscriptionTopicAgentTest.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.subscription.agent; + +import org.apache.iotdb.commons.subscription.meta.topic.TopicMeta; +import org.apache.iotdb.rpc.subscription.config.TopicConstant; + +import org.junit.Assert; +import org.junit.Test; + +import java.util.HashMap; +import java.util.Map; + +public class SubscriptionTopicAgentTest { + + @Test + public void testColumnFilterRefreshSkippedForOwnerOnlyUpdate() { + final TopicMeta oldMeta = createTableTopicMeta("column_name = \"temperature\"", "db", "t1"); + final Map updatedAttributes = new HashMap<>(); + updatedAttributes.put(TopicConstant.OWNER_ID_KEY, "owner-1"); + updatedAttributes.put(TopicConstant.OWNER_EPOCH_KEY, "1"); + + Assert.assertFalse( + SubscriptionTopicAgent.shouldRefreshColumnFilter( + oldMeta, oldMeta.deepCopyWithUpdatedAttributes(updatedAttributes))); + } + + @Test + public void testColumnFilterRefreshTriggeredForBindingInputUpdate() { + final TopicMeta oldMeta = createTableTopicMeta("column_name = \"temperature\"", "db", "t1"); + + Assert.assertTrue( + SubscriptionTopicAgent.shouldRefreshColumnFilter( + oldMeta, createTableTopicMeta("column_name = \"status\"", "db", "t1"))); + Assert.assertTrue( + SubscriptionTopicAgent.shouldRefreshColumnFilter( + oldMeta, createTableTopicMeta("column_name = \"temperature\"", "db2", "t1"))); + Assert.assertTrue( + SubscriptionTopicAgent.shouldRefreshColumnFilter( + oldMeta, createTableTopicMeta("column_name = \"temperature\"", "db", "t2"))); + } + + @Test + public void testColumnFilterRefreshTriggeredForNewTableTopicOnly() { + Assert.assertTrue( + SubscriptionTopicAgent.shouldRefreshColumnFilter( + null, createTableTopicMeta("column_name = \"temperature\"", "db", "t1"))); + Assert.assertFalse( + SubscriptionTopicAgent.shouldRefreshColumnFilter(null, createTreeTopicMeta())); + } + + private static TopicMeta createTableTopicMeta( + final String columnFilter, final String database, final String table) { + final Map attributes = new HashMap<>(); + attributes.put("__system.sql-dialect", "table"); + attributes.put(TopicConstant.COLUMN_FILTER_KEY, columnFilter); + attributes.put(TopicConstant.DATABASE_KEY, database); + attributes.put(TopicConstant.TABLE_KEY, table); + return new TopicMeta("topic", 1L, attributes); + } + + private static TopicMeta createTreeTopicMeta() { + final Map attributes = new HashMap<>(); + attributes.put("__system.sql-dialect", "tree"); + attributes.put(TopicConstant.PATH_KEY, "root.**"); + return new TopicMeta("topic", 1L, attributes); + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusLogToTabletConverterTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusLogToTabletConverterTest.java index 5cc56bb016222..b2684ed7ff011 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusLogToTabletConverterTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusLogToTabletConverterTest.java @@ -28,12 +28,15 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsOfOneDeviceNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertTabletNode; import org.apache.iotdb.db.queryengine.plan.statement.StatementTestUtils; +import org.apache.iotdb.db.subscription.columnfilter.ColumnFilterMatcher; import org.apache.tsfile.enums.ColumnCategory; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.utils.BitMap; import org.apache.tsfile.write.record.Tablet; import org.apache.tsfile.write.schema.MeasurementSchema; import org.junit.Assert; @@ -41,8 +44,9 @@ import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; import java.util.List; -import java.util.regex.Pattern; public class ConsensusLogToTabletConverterTest { @@ -66,7 +70,7 @@ public void testConvertRelationalInsertRowNodeWithSingleMatchedColumn() { @Test public void testConvertRelationalInsertRowNodeWithMultipleMatchedColumns() { - final ConsensusLogToTabletConverter converter = createConverter("(id1|m1)"); + final ConsensusLogToTabletConverter converter = createConverter("id1", "m1"); final List tablets = converter.convert(StatementTestUtils.genInsertRowNode(9)); @@ -107,13 +111,28 @@ public void testConvertRelationalInsertTabletNodeWithSingleMatchedColumn() { } @Test - public void testConvertRelationalInsertTabletNodeSkipsNullMatchedFieldColumn() { + public void testConvertRelationalInsertTabletNodeKeepsNullMatchedFieldColumn() { final ConsensusLogToTabletConverter converter = createConverter("m1"); final RelationalInsertTabletNode node = StatementTestUtils.genInsertTabletNode(3, 10); node.getColumns()[2] = null; + final BitMap[] bitMaps = new BitMap[] {null, null, new BitMap(3)}; + bitMaps[2].mark(0); + bitMaps[2].mark(1); + bitMaps[2].mark(2); + node.setBitMaps(bitMaps); - Assert.assertTrue(converter.convert(node).isEmpty()); + final List tablets = converter.convert(node); + + Assert.assertEquals(1, tablets.size()); + final Tablet tablet = tablets.get(0); + Assert.assertEquals(2, tablet.getSchemas().size()); + Assert.assertEquals("id1", tablet.getSchemas().get(0).getMeasurementName()); + Assert.assertEquals("m1", tablet.getSchemas().get(1).getMeasurementName()); + Assert.assertNull(tablet.getValues()[1]); + Assert.assertTrue(tablet.getBitMaps()[1].isMarked(0)); + Assert.assertTrue(tablet.getBitMaps()[1].isMarked(1)); + Assert.assertTrue(tablet.getBitMaps()[1].isMarked(2)); } @Test @@ -150,6 +169,23 @@ public void testConvertRelationalInsertRowNodeKeepsTagColumnsForMatchedField() { Assert.assertEquals(11.0, ((double[]) tablet.getValues()[1])[0], 0.0); } + @Test + public void testConvertRelationalInsertRowNodeKeepsNullMatchedFieldWithBitmap() { + final ConsensusLogToTabletConverter converter = createConverter("m1"); + final RelationalInsertRowNode node = StatementTestUtils.genInsertRowNode(12); + node.getValues()[2] = null; + + final List tablets = converter.convert(node); + + Assert.assertEquals(1, tablets.size()); + final Tablet tablet = tablets.get(0); + Assert.assertEquals(2, tablet.getSchemas().size()); + Assert.assertEquals("id1", tablet.getSchemas().get(0).getMeasurementName()); + Assert.assertEquals("m1", tablet.getSchemas().get(1).getMeasurementName()); + Assert.assertEquals("id:12", toUtf8(((Binary[]) tablet.getValues()[0])[0])); + Assert.assertTrue(tablet.getBitMaps()[1].isMarked(0)); + } + @Test public void testConvertRelationalInsertNodeReturnsEmptyWhenNoColumnsMatch() { final ConsensusLogToTabletConverter converter = createConverter("not_exist"); @@ -235,11 +271,11 @@ public void testConvertTreeInsertTabletNodeSkipsNullColumn() throws IllegalPathE Assert.assertArrayEquals(new int[] {1, 2}, (int[]) tablet.getValues()[0]); } - private static ConsensusLogToTabletConverter createConverter(final String columnPattern) { + private static ConsensusLogToTabletConverter createConverter(final String... selectedColumns) { return new ConsensusLogToTabletConverter( null, new TablePattern(true, DATABASE_NAME, StatementTestUtils.tableName()), - Pattern.compile(columnPattern), + ColumnFilterMatcher.ofSelectedColumnNames(new HashSet<>(Arrays.asList(selectedColumns))), DATABASE_NAME); } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/columnfilter/ColumnFilterBinderTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/columnfilter/ColumnFilterBinderTest.java new file mode 100644 index 0000000000000..d1052f012adb7 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/columnfilter/ColumnFilterBinderTest.java @@ -0,0 +1,234 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.subscription.columnfilter; + +import org.apache.iotdb.commons.schema.table.TreeViewSchema; +import org.apache.iotdb.commons.schema.table.TsTable; +import org.apache.iotdb.commons.schema.table.column.AttributeColumnSchema; +import org.apache.iotdb.commons.schema.table.column.FieldColumnSchema; +import org.apache.iotdb.commons.schema.table.column.TagColumnSchema; +import org.apache.iotdb.commons.schema.table.column.TimeColumnSchema; +import org.apache.iotdb.rpc.subscription.config.TopicConfig; +import org.apache.iotdb.rpc.subscription.config.TopicConstant; + +import org.apache.tsfile.enums.ColumnCategory; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.write.record.Tablet; +import org.junit.Assert; +import org.junit.Test; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +public class ColumnFilterBinderTest { + + @Test + public void testBindSnapshotKeepsOnlyBoundTagsAtRuntime() { + final BoundColumnFilter boundFilter = + new ColumnFilterBinder() + .bind( + createTableTopicConfig("column_name = \"temperature\""), + Collections.singletonMap( + "db", Collections.singletonMap("sensors", createTableSchema()))); + final ColumnFilterMatcher matcher = ColumnFilterMatcher.fromBoundColumnFilter(boundFilter); + + final Tablet prunedTablet = + TabletColumnPruner.pruneTableModelTablet(createRuntimeTabletWithNewTag(), "db", matcher); + + Assert.assertNotNull(prunedTablet); + Assert.assertEquals(2, prunedTablet.getSchemas().size()); + Assert.assertEquals("device", prunedTablet.getSchemas().get(0).getMeasurementName()); + Assert.assertEquals("temperature", prunedTablet.getSchemas().get(1).getMeasurementName()); + } + + @Test + public void testAttributeMatchBindsTagsButNotAttribute() { + final BoundColumnFilter boundFilter = + new ColumnFilterBinder() + .bind( + createTableTopicConfig("category = \"ATTRIBUTE\""), + Collections.singletonMap( + "db", Collections.singletonMap("sensors", createTableSchema()))); + final ColumnFilterMatcher matcher = ColumnFilterMatcher.fromBoundColumnFilter(boundFilter); + + final Tablet prunedTablet = + TabletColumnPruner.pruneTableModelTablet(createRuntimeTabletWithNewTag(), "db", matcher); + + Assert.assertNotNull(prunedTablet); + Assert.assertEquals(1, prunedTablet.getSchemas().size()); + Assert.assertEquals("device", prunedTablet.getSchemas().get(0).getMeasurementName()); + Assert.assertEquals(ColumnCategory.TAG, prunedTablet.getColumnTypes().get(0)); + } + + @Test + public void testBindTimeSelectionPerTable() { + final Map tables = new HashMap<>(); + tables.put("sensors", createTableSchema("sensors", "time")); + tables.put("events", createTableSchema("events", "event_time")); + final BoundColumnFilter boundFilter = + new ColumnFilterBinder() + .bind( + createTableTopicConfig("column_name = \"time\""), + Collections.singletonMap("db", tables)); + final ColumnFilterMatcher matcher = ColumnFilterMatcher.fromBoundColumnFilter(boundFilter); + + Assert.assertTrue(boundFilter.isTimeSelected()); + Assert.assertTrue(matcher.isTimeSelected("db", "sensors")); + Assert.assertFalse(matcher.isTimeSelected("db", "events")); + Assert.assertFalse(matcher.match("db", "new_sensors", "time")); + Assert.assertEquals( + Boolean.TRUE, matcher.getTimeSelectedByTable("db").get("db").get("sensors")); + Assert.assertEquals( + Boolean.FALSE, matcher.getTimeSelectedByTable("db").get("db").get("events")); + Assert.assertTrue(matcher.getTimeSelectedByTable("db2").isEmpty()); + } + + @Test + public void testBindTreeViewUsesViewColumnsAndSelectsSourceColumns() { + final BoundColumnFilter boundFilter = + new ColumnFilterBinder() + .bind( + createTableTopicConfig("column_name = \"temp_alias\""), + Collections.singletonMap( + "db", Collections.singletonMap("sensors_view", createTreeViewSchema()))); + final ColumnFilterMatcher matcher = ColumnFilterMatcher.fromBoundColumnFilter(boundFilter); + + Assert.assertTrue(matcher.match("db", "sensors_view", "temperature")); + Assert.assertFalse(matcher.match("db", "sensors_view", "temp_alias")); + Assert.assertTrue(matcher.match("db", "sensors_view", "device_alias")); + Assert.assertFalse(matcher.match("db", "sensors_view", "device")); + + final Tablet prunedTablet = + TabletColumnPruner.pruneTableModelTablet(createRuntimeViewTablet(), "db", matcher); + + Assert.assertNotNull(prunedTablet); + Assert.assertEquals(2, prunedTablet.getSchemas().size()); + Assert.assertEquals("device_alias", prunedTablet.getSchemas().get(0).getMeasurementName()); + Assert.assertEquals("temperature", prunedTablet.getSchemas().get(1).getMeasurementName()); + } + + @Test + public void testRuntimeExpressionTimeSelectionUsesTimestampDatatype() { + final ColumnFilterMatcher matcher = + ColumnFilterMatcher.fromTopicConfig(createTableTopicConfig("datatype = \"TIMESTAMP\"")); + + Assert.assertTrue(matcher.isTimeSelected()); + Assert.assertTrue(matcher.isTimeSelected("db", "sensors")); + } + + @Test + public void testRuntimeExpressionGlobalTimeSelectionCanBeFalse() { + final ColumnFilterMatcher matcher = + ColumnFilterMatcher.fromTopicConfig( + createTableTopicConfig("column_name = \"temperature\"")); + + Assert.assertFalse(matcher.isTimeSelected()); + Assert.assertFalse(matcher.isTimeSelected("db", "sensors")); + } + + private static TopicConfig createTableTopicConfig(final String columnFilter) { + final Map attributes = new HashMap<>(); + attributes.put("__system.sql-dialect", "table"); + attributes.put(TopicConstant.COLUMN_FILTER_KEY, columnFilter); + return new TopicConfig(attributes); + } + + private static TsTable createTableSchema() { + return createTableSchema("sensors", "time"); + } + + private static TsTable createTableSchema(final String tableName, final String timeColumnName) { + final TsTable table = new TsTable(tableName); + table.addColumnSchema(new TimeColumnSchema(timeColumnName, TSDataType.TIMESTAMP)); + table.addColumnSchema(new TagColumnSchema("device", TSDataType.STRING)); + table.addColumnSchema(new AttributeColumnSchema("site", TSDataType.STRING)); + table.addColumnSchema(new FieldColumnSchema("temperature", TSDataType.DOUBLE)); + table.addColumnSchema(new FieldColumnSchema("status", TSDataType.STRING)); + return table; + } + + private static TsTable createTreeViewSchema() { + final TsTable table = new TsTable("sensors_view"); + table.addProp(TreeViewSchema.TREE_PATH_PATTERN, "root.test.**"); + table.addColumnSchema(new TimeColumnSchema("time", TSDataType.TIMESTAMP)); + final TagColumnSchema tagColumnSchema = new TagColumnSchema("device_alias", TSDataType.STRING); + table.addColumnSchema(tagColumnSchema); + table.addColumnSchema(new AttributeColumnSchema("site", TSDataType.STRING)); + final FieldColumnSchema temperature = new FieldColumnSchema("temp_alias", TSDataType.DOUBLE); + TreeViewSchema.setOriginalName(temperature, "temperature"); + table.addColumnSchema(temperature); + final FieldColumnSchema status = new FieldColumnSchema("status_alias", TSDataType.STRING); + TreeViewSchema.setOriginalName(status, "status"); + table.addColumnSchema(status); + return table; + } + + private static Tablet createRuntimeTabletWithNewTag() { + final Tablet tablet = + new Tablet( + "sensors", + Arrays.asList("device", "new_tag", "site", "temperature", "status"), + Arrays.asList( + TSDataType.STRING, + TSDataType.STRING, + TSDataType.STRING, + TSDataType.DOUBLE, + TSDataType.STRING), + Arrays.asList( + ColumnCategory.TAG, + ColumnCategory.TAG, + ColumnCategory.ATTRIBUTE, + ColumnCategory.FIELD, + ColumnCategory.FIELD), + 1); + tablet.addTimestamp(0, 1L); + tablet.addValue(0, 0, "d1"); + tablet.addValue(0, 1, "new"); + tablet.addValue(0, 2, "north"); + tablet.addValue(0, 3, 36.5); + tablet.addValue(0, 4, "ok"); + tablet.setRowSize(1); + return tablet; + } + + private static Tablet createRuntimeViewTablet() { + final Tablet tablet = + new Tablet( + "sensors_view", + Arrays.asList("device_alias", "site", "temperature", "status"), + Arrays.asList( + TSDataType.STRING, TSDataType.STRING, TSDataType.DOUBLE, TSDataType.STRING), + Arrays.asList( + ColumnCategory.TAG, + ColumnCategory.ATTRIBUTE, + ColumnCategory.FIELD, + ColumnCategory.FIELD), + 1); + tablet.addTimestamp(0, 1L); + tablet.addValue(0, 0, "d1"); + tablet.addValue(0, 1, "north"); + tablet.addValue(0, 2, 36.5); + tablet.addValue(0, 3, "ok"); + tablet.setRowSize(1); + return tablet; + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/columnfilter/ColumnFilterParserTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/columnfilter/ColumnFilterParserTest.java new file mode 100644 index 0000000000000..d66f80dacfc77 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/columnfilter/ColumnFilterParserTest.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.subscription.columnfilter; + +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.Expression; +import org.apache.iotdb.commons.subscription.columnfilter.ColumnMetadata; +import org.apache.iotdb.rpc.subscription.exception.SubscriptionException; + +import org.junit.Assert; +import org.junit.Test; + +public class ColumnFilterParserTest { + + private static final ColumnFilterParser PARSER = new ColumnFilterParser(); + private static final ColumnMetadata TEMPERATURE = + new ColumnMetadata("db1", "weather", "temperature", "DOUBLE", "FIELD"); + private static final ColumnMetadata TAG = + new ColumnMetadata("db1", "weather", "city", "STRING", "TAG"); + + @Test + public void testBooleanLiteral() throws Exception { + Assert.assertTrue(evaluate("true", TEMPERATURE)); + Assert.assertFalse(evaluate("false", TEMPERATURE)); + } + + @Test + public void testComparisonAndInAreCaseInsensitive() throws Exception { + Assert.assertTrue( + evaluate("CATEGORY = \"field\" AND datatype IN (\"int32\", \"double\")", TEMPERATURE)); + Assert.assertTrue(evaluate("\"column_name\" != \"humidity\"", TEMPERATURE)); + Assert.assertTrue(evaluate("column_name <> \"humidity\"", TEMPERATURE)); + Assert.assertFalse(evaluate("table_name NOT IN (\"weather\", \"status\")", TEMPERATURE)); + } + + @Test + public void testLikeRegexpAndNot() throws Exception { + Assert.assertTrue(evaluate("column_name LIKE \"temp%\"", TEMPERATURE)); + Assert.assertTrue(evaluate("column_name REGEXP \"temp.*\"", TEMPERATURE)); + Assert.assertTrue(evaluate("NOT category = \"TAG\"", TEMPERATURE)); + Assert.assertTrue(evaluate("column_name NOT LIKE \"hum%\"", TEMPERATURE)); + Assert.assertTrue(evaluate("column_name NOT REGEXP \"hum.*\"", TEMPERATURE)); + } + + @Test + public void testEscapedStringLiteralAndLikeEscape() throws Exception { + final ColumnMetadata quotedColumn = + new ColumnMetadata("db1", "weather", "temperature\"sensor", "DOUBLE", "FIELD"); + final ColumnMetadata escapedLikeColumn = + new ColumnMetadata("db1", "weather", "temp_sensor", "DOUBLE", "FIELD"); + + Assert.assertTrue(evaluate("column_name = \"temperature\"\"sensor\"", quotedColumn)); + Assert.assertTrue(evaluate("column_name LIKE \"temp!_%\" ESCAPE \"!\"", escapedLikeColumn)); + } + + @Test + public void testLogicalPrecedence() throws Exception { + Assert.assertTrue( + evaluate( + "category = \"TAG\" OR category = \"FIELD\" AND datatype = \"DOUBLE\"", TEMPERATURE)); + Assert.assertFalse( + evaluate( + "(category = \"TAG\" OR category = \"FIELD\") AND datatype = \"STRING\"", TEMPERATURE)); + } + + @Test + public void testIsNullAndIsNotNull() throws Exception { + Assert.assertFalse(evaluate("column_name IS NULL", TAG)); + Assert.assertTrue(evaluate("column_name IS NOT NULL", TAG)); + } + + @Test + public void testRejectInvalidExpressions() { + assertRejected("temperature > 10", "unsupported comparison operator"); + assertRejected("column_name = \"s1\" + \"s2\"", "unexpected character '+'"); + assertRejected("lower(column_name) = \"s1\"", "expected column predicate operator"); + assertRejected("column_name = table_name", "expected string literal"); + assertRejected("owner = \"alice\"", "unsupported column metadata field"); + assertRejected("column_name", "expected column predicate operator"); + assertRejected("\"temperature\"", "expected column predicate operator"); + assertRejected("column_name REGEXP \"[\"", "illegal REGEXP pattern"); + assertRejected( + "column_name LIKE \"temp!\" ESCAPE \"!\"", "LIKE pattern ends with escape character"); + } + + private static boolean evaluate(final String expression, final ColumnMetadata metadata) + throws SubscriptionException { + final Expression parsed = PARSER.parseAndValidate(expression); + return ColumnFilterEvaluator.evaluate(parsed, metadata); + } + + private static void assertRejected(final String expression, final String expectedMessagePart) { + try { + PARSER.parseAndValidate(expression); + Assert.fail("Expected column-filter to be rejected: " + expression); + } catch (final SubscriptionException e) { + Assert.assertTrue(e.getMessage(), e.getMessage().contains(expectedMessagePart)); + } + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/columnfilter/TabletColumnPrunerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/columnfilter/TabletColumnPrunerTest.java new file mode 100644 index 0000000000000..46984f62436fc --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/columnfilter/TabletColumnPrunerTest.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.subscription.columnfilter; + +import org.apache.iotdb.rpc.subscription.config.TopicConfig; +import org.apache.iotdb.rpc.subscription.config.TopicConstant; + +import org.apache.tsfile.enums.ColumnCategory; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.write.record.Tablet; +import org.junit.Assert; +import org.junit.Test; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class TabletColumnPrunerTest { + + @Test + public void testPruneKeepsTagsAndDropsAttributes() { + final Tablet tablet = createTablet(); + final ColumnFilterMatcher matcher = + ColumnFilterMatcher.fromTopicConfig( + createTableTopicConfig("column_name = \"temperature\" or category = \"ATTRIBUTE\"")); + + final Tablet pruned = TabletColumnPruner.pruneTableModelTablet(tablet, "db", matcher); + + Assert.assertNotNull(pruned); + Assert.assertEquals(2, pruned.getSchemas().size()); + Assert.assertEquals("device", pruned.getSchemas().get(0).getMeasurementName()); + Assert.assertEquals("temperature", pruned.getSchemas().get(1).getMeasurementName()); + Assert.assertEquals(ColumnCategory.TAG, pruned.getColumnTypes().get(0)); + Assert.assertEquals(ColumnCategory.FIELD, pruned.getColumnTypes().get(1)); + Assert.assertSame(tablet.getValues()[0], pruned.getValues()[0]); + Assert.assertSame(tablet.getValues()[2], pruned.getValues()[1]); + } + + @Test + public void testPruneReturnsNullWhenNoColumnMatches() { + final ColumnFilterMatcher matcher = + ColumnFilterMatcher.fromTopicConfig(createTableTopicConfig("column_name = \"missing\"")); + + Assert.assertNull(TabletColumnPruner.pruneTableModelTablet(createTablet(), "db", matcher)); + } + + @Test + public void testPruneKeepsMatchedNullColumnArray() { + final Tablet tablet = createTablet(); + tablet.getValues()[2] = null; + tablet.initBitMaps(); + tablet.getBitMaps()[2].mark(0); + final ColumnFilterMatcher matcher = + ColumnFilterMatcher.fromTopicConfig( + createTableTopicConfig("column_name = \"temperature\"")); + + final Tablet pruned = TabletColumnPruner.pruneTableModelTablet(tablet, "db", matcher); + + Assert.assertNotNull(pruned); + Assert.assertEquals(2, pruned.getSchemas().size()); + Assert.assertEquals("device", pruned.getSchemas().get(0).getMeasurementName()); + Assert.assertEquals("temperature", pruned.getSchemas().get(1).getMeasurementName()); + Assert.assertNull(pruned.getValues()[1]); + Assert.assertTrue(pruned.getBitMaps()[1].isMarked(0)); + } + + private static TopicConfig createTableTopicConfig(final String columnFilter) { + final Map attributes = new HashMap<>(); + attributes.put("__system.sql-dialect", "table"); + attributes.put(TopicConstant.COLUMN_FILTER_KEY, columnFilter); + return new TopicConfig(attributes); + } + + private static Tablet createTablet() { + final List columnNames = Arrays.asList("device", "site", "temperature", "status"); + final List dataTypes = + Arrays.asList(TSDataType.STRING, TSDataType.STRING, TSDataType.DOUBLE, TSDataType.STRING); + final List categories = + Arrays.asList( + ColumnCategory.TAG, + ColumnCategory.ATTRIBUTE, + ColumnCategory.FIELD, + ColumnCategory.FIELD); + final Tablet tablet = new Tablet("sensors", columnNames, dataTypes, categories, 1); + tablet.addTimestamp(0, 1L); + tablet.addValue(0, 0, "d1"); + tablet.addValue(0, 1, "north"); + tablet.addValue(0, 2, 36.5); + tablet.addValue(0, 3, "ok"); + tablet.setRowSize(1); + return tablet; + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/event/batch/SubscriptionPipeTsFileEventBatchTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/event/batch/SubscriptionPipeTsFileEventBatchTest.java new file mode 100644 index 0000000000000..c983f6a83104b --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/event/batch/SubscriptionPipeTsFileEventBatchTest.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.subscription.event.batch; + +import org.apache.iotdb.db.pipe.event.common.tablet.PipeRawTabletInsertionEvent; +import org.apache.iotdb.db.subscription.broker.SubscriptionPrefetchingTsFileQueue; +import org.apache.iotdb.db.subscription.event.SubscriptionEvent; + +import org.apache.tsfile.enums.ColumnCategory; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.write.record.Tablet; +import org.junit.Assert; +import org.junit.Test; + +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.atomic.AtomicLong; + +public class SubscriptionPipeTsFileEventBatchTest { + + @Test + public void testEmptyInnerBatchConsumesOuterBatchWithoutPayload() throws Exception { + final TestSubscriptionPipeTsFileEventBatch batch = + new TestSubscriptionPipeTsFileEventBatch( + new SubscriptionPrefetchingTsFileQueue("broker", "topic", null, new AtomicLong())); + batch.addEnrichedEvent( + new PipeRawTabletInsertionEvent( + true, + "root.db", + "db", + "root.db", + createTablet(), + false, + "pipe", + 1L, + null, + null, + false)); + + Assert.assertTrue(batch.shouldEmitForTest()); + + final List events = batch.generateSubscriptionEventsForTest(); + Assert.assertNotNull(events); + Assert.assertTrue(events.isEmpty()); + Assert.assertEquals(0, batch.getPipeEventCount()); + + batch.cleanUp(true); + } + + private static Tablet createTablet() { + final Tablet tablet = + new Tablet( + "sensors", + Arrays.asList("device", "temperature"), + Arrays.asList(TSDataType.STRING, TSDataType.DOUBLE), + Arrays.asList(ColumnCategory.TAG, ColumnCategory.FIELD), + 1); + tablet.addTimestamp(0, 1L); + tablet.addValue(0, 0, "d1"); + tablet.addValue(0, 1, 36.5); + tablet.setRowSize(1); + return tablet; + } + + private static class TestSubscriptionPipeTsFileEventBatch + extends SubscriptionPipeTsFileEventBatch { + + private TestSubscriptionPipeTsFileEventBatch( + final SubscriptionPrefetchingTsFileQueue prefetchingQueue) { + super(1, prefetchingQueue, 100_000, 1024 * 1024); + } + + private void addEnrichedEvent(final PipeRawTabletInsertionEvent event) { + enrichedEvents.add(event); + } + + private boolean shouldEmitForTest() { + return shouldEmit(); + } + + private List generateSubscriptionEventsForTest() throws Exception { + return generateSubscriptionEvents(); + } + } +} diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/subscription/columnfilter/ColumnMetadata.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/subscription/columnfilter/ColumnMetadata.java new file mode 100644 index 0000000000000..9c43d61cf1bb9 --- /dev/null +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/subscription/columnfilter/ColumnMetadata.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.subscription.columnfilter; + +import java.util.Objects; + +public class ColumnMetadata { + + private final String database; + private final String tableName; + private final String columnName; + private final String datatype; + private final String category; + + public ColumnMetadata( + final String database, + final String tableName, + final String columnName, + final String datatype, + final String category) { + this.database = Objects.requireNonNull(database, "database should not be null"); + this.tableName = Objects.requireNonNull(tableName, "tableName should not be null"); + this.columnName = Objects.requireNonNull(columnName, "columnName should not be null"); + this.datatype = Objects.requireNonNull(datatype, "datatype should not be null"); + this.category = Objects.requireNonNull(category, "category should not be null"); + } + + public String getDatabase() { + return database; + } + + public String getTableName() { + return tableName; + } + + public String getColumnName() { + return columnName; + } + + public String getDatatype() { + return datatype; + } + + public String getCategory() { + return category; + } +} diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/subscription/meta/topic/TopicMeta.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/subscription/meta/topic/TopicMeta.java index c44692f63588d..de01a55e2bf4d 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/subscription/meta/topic/TopicMeta.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/subscription/meta/topic/TopicMeta.java @@ -453,6 +453,8 @@ public Map generateExtractorAttributes( if (config.isTableTopic()) { // table model: database name and table name extractorAttributes.putAll(config.getAttributesWithSourceDatabaseAndTableName()); + // column-filter is evaluated by subscription runtime on DataNode. + extractorAttributes.putAll(config.getAttributesWithSourceColumnFilter()); } else { // tree model: path or pattern extractorAttributes.putAll(config.getAttributesWithSourcePathOrPattern()); diff --git a/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/ColumnFilter.g4 b/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/ColumnFilter.g4 new file mode 100644 index 0000000000000..ca956ca639ea8 --- /dev/null +++ b/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/ColumnFilter.g4 @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +grammar ColumnFilter; + +options { caseInsensitive = true; } + +columnFilter + : booleanExpression EOF + ; + +booleanExpression + : NOT booleanExpression #logicalNot + | booleanExpression AND booleanExpression #logicalBinary + | booleanExpression OR booleanExpression #logicalBinary + | predicate #predicateExpression + ; + +predicate + : booleanValue + | field comparisonOperator string + | field NOT? IN '(' string (',' string)* ')' + | field NOT? LIKE string (ESCAPE string)? + | field NOT? REGEXP string + | field IS NOT? NULL + | '(' booleanExpression ')' + ; + +field + : IDENTIFIER + | QUOTED_IDENTIFIER + ; + +booleanValue + : TRUE + | FALSE + ; + +comparisonOperator + : EQ + | NEQ + ; + +string + : QUOTED_IDENTIFIER + ; + +TRUE: 'TRUE'; +FALSE: 'FALSE'; +AND: 'AND'; +OR: 'OR'; +NOT: 'NOT'; +IN: 'IN'; +LIKE: 'LIKE'; +REGEXP: 'REGEXP'; +IS: 'IS'; +NULL: 'NULL'; +ESCAPE: 'ESCAPE'; +EQ: '='; +NEQ: '!=' | '<>'; + +IDENTIFIER + : [A-Z_] [A-Z_0-9]* + ; + +QUOTED_IDENTIFIER + : '"' ('""' | ~'"')* '"' + ; + +WS + : [ \r\n\t]+ -> channel(HIDDEN) + ;