diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java index a354af647418b9..c038fcd0e24b04 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java @@ -2806,8 +2806,12 @@ private boolean processAddIndex(CreateIndexClause alterClause, OlapTable olapTab } if (indexDef.isAnnIndex()) { - if (olapTable.getKeysType() != KeysType.DUP_KEYS) { - throw new AnalysisException("ANN index can only be built on table with DUP_KEYS"); + if (olapTable.getKeysType() != KeysType.DUP_KEYS + && !(olapTable.getKeysType() == KeysType.UNIQUE_KEYS + && olapTable.getEnableUniqueKeyMergeOnWrite())) { + throw new AnalysisException( + "ANN index can only be built on table with DUP_KEYS or UNIQUE_KEYS" + + " with merge-on-write enabled"); } AnnIndexPropertiesChecker.checkProperties(indexDef.getProperties()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java index d64b65bc64a279..abcbe848582486 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java @@ -255,8 +255,11 @@ public void checkColumn(Column column, KeysType keysType, boolean enableUniqueKe if (!itemType.isFloatingPointType()) { throw new AnalysisException("ANN index column item type must be float type"); } - if (keysType != KeysType.DUP_KEYS) { - throw new AnalysisException("ANN index can only be used in DUP_KEYS table"); + if (keysType != KeysType.DUP_KEYS + && !(keysType == KeysType.UNIQUE_KEYS && enableUniqueKeyMergeOnWrite)) { + throw new AnalysisException( + "ANN index can only be used in DUP_KEYS table or UNIQUE_KEYS table with" + + " merge-on-write enabled"); } if (invertedIndexFileStorageFormat == TInvertedIndexFileStorageFormat.V1) { throw new AnalysisException("ANN index is not supported in index format V1"); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/IndexDefinition.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/IndexDefinition.java index 1d151801f92d84..8ab371db208f21 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/IndexDefinition.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/IndexDefinition.java @@ -148,8 +148,11 @@ public void checkColumn(ColumnDefinition column, KeysType keysType, if (!itemType.isFloatType()) { throw new AnalysisException("ANN index column item type must be float type, invalid index: " + name); } - if (keysType != KeysType.DUP_KEYS) { - throw new AnalysisException("ANN index can only be used in DUP_KEYS table"); + if (keysType != KeysType.DUP_KEYS + && !(keysType == KeysType.UNIQUE_KEYS && enableUniqueKeyMergeOnWrite)) { + throw new AnalysisException( + "ANN index can only be used in DUP_KEYS table or UNIQUE_KEYS table with" + + " merge-on-write enabled"); } return; } diff --git a/regression-test/data/ann_index_p0/ann_index_on_mow.out b/regression-test/data/ann_index_p0/ann_index_on_mow.out new file mode 100644 index 00000000000000..d0e14b53133114 --- /dev/null +++ b/regression-test/data/ann_index_p0/ann_index_on_mow.out @@ -0,0 +1,31 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql_1 -- +1 [1, 2, 3] 11 +2 [4, 5, 6] 22 +3 [7, 8, 9] 33 + +-- !sql_2 -- +1 0.0 + +-- !sql_3 -- +1 [10, 20, 30] 111 +2 [40, 50, 60] 222 +3 [70, 80, 90] 333 + +-- !sql_4 -- +1 0.0 + +-- !sql_5 -- +1 [10, 20, 30] 111 +2 [40, 50, 60] 222 +3 [70, 80, 90] 333 + +-- !sql_6 -- + +-- !sql_7 -- +1 [10, 20, 30] \N 1 4 +2 [40, 50, 60] \N 1 4 +3 [70, 80, 90] \N 1 4 + +-- !sql_8 -- + diff --git a/regression-test/suites/ann_index_p0/ann_index_on_mow.groovy b/regression-test/suites/ann_index_p0/ann_index_on_mow.groovy new file mode 100644 index 00000000000000..a14a116c5337fd --- /dev/null +++ b/regression-test/suites/ann_index_p0/ann_index_on_mow.groovy @@ -0,0 +1,64 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("ann_index_on_mow") { + sql "drop table if exists ann_index_on_mow" + test { + sql """ + CREATE TABLE ann_index_on_mow ( + id INT NOT NULL COMMENT "", + vec ARRAY NOT NULL COMMENT "", + value INT NULL COMMENT "", + INDEX ann_idx (vec) USING ANN PROPERTIES( + "index_type" = "hnsw", + "metric_type" = "l2_distance", + "dim" = "3" + ) + ) ENGINE=OLAP + UNIQUE KEY(id) COMMENT "OLAP" + DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "enable_unique_key_merge_on_write" = "true" + ); + """ + } + + sql "insert into ann_index_on_mow values (1, [1.0, 2.0, 3.0], 11),(2, [4.0, 5.0, 6.0], 22),(3, [7.0, 8.0, 9.0], 33)" + + qt_sql_1 "select * from ann_index_on_mow order by id" + + qt_sql_2 "select id, l2_distance_approximate(vec, [1.0, 2.0, 3.0]) as dist from ann_index_on_mow order by dist limit 1;" + + sql "insert into ann_index_on_mow values (1, [10.0, 20.0, 30.0], 111),(2, [40.0, 50.0, 60.0], 222),(3, [70.0, 80.0, 90.0], 333)" + + qt_sql_3 "select * from ann_index_on_mow order by id" + + qt_sql_4 "select id, l2_distance_approximate(vec, [10.0, 20.0, 30.0]) as dist from ann_index_on_mow order by dist limit 1;" + + qt_sql_5 "select * from ann_index_on_mow order by id" + + sql "insert into ann_index_on_mow (id, vec, __DORIS_DELETE_SIGN__) values (1, [10.0, 20.0, 30.0], 1),(2, [40.0, 50.0, 60.0], 1),(3, [70.0, 80.0, 90.0], 1);" + + qt_sql_6 "select * from ann_index_on_mow order by id" + + sql "set show_hidden_columns=true;" + qt_sql_7 "select * from ann_index_on_mow order by id" + sql "set show_hidden_columns=false;" + + qt_sql_8 "select id, l2_distance_approximate(vec, [10.0, 20.0, 30.0]) as dist from ann_index_on_mow order by dist limit 1;" +} diff --git a/regression-test/suites/ann_index_p0/build_ann_index_test.groovy b/regression-test/suites/ann_index_p0/build_ann_index_test.groovy index e8de0d3d2d1702..26101a60950a1c 100644 --- a/regression-test/suites/ann_index_p0/build_ann_index_test.groovy +++ b/regression-test/suites/ann_index_p0/build_ann_index_test.groovy @@ -109,4 +109,49 @@ suite("build_ann_index_test") { // BUILD INDEX sql "BUILD INDEX idx_test_ann ON table_build_ann_index_test;" wait_for_last_build_index_on_table_finish(tableName, timeout) + + + // case 2: create and build ann index on mow table + sql "drop table if exists table_build_ann_index_on_mow;" + + sql """ + CREATE TABLE `table_build_ann_index_on_mow` ( + `id` int NOT NULL COMMENT "", + `embedding` array NOT NULL COMMENT "" + ) ENGINE=OLAP + UNIQUE KEY(`id`) COMMENT "OLAP" + DISTRIBUTED BY HASH(`id`) BUCKETS 2 + PROPERTIES ( + "replication_num" = "1", + "enable_unique_key_merge_on_write" = "true" + ); + """ + + sql """ + INSERT INTO table_build_ann_index_on_mow (id, embedding) VALUES + (0, [39.906116, 10.495334, 54.08394, 88.67262, 55.243687, 10.162686, 36.335983, 38.684258]), + (1, [62.759315, 97.15586, 25.832521, 39.604908, 88.76715, 72.64085, 9.688437, 17.721428]), + (2, [15.447449, 59.7771, 65.54516, 12.973712, 99.685135, 72.080734, 85.71118, 99.35976]), + (3, [72.26747, 46.42257, 32.368374, 80.50209, 5.777631, 98.803314, 7.0915947, 68.62693]), + (4, [22.098177, 74.10027, 63.634556, 4.710955, 12.405106, 79.39356, 63.014366, 68.67834]), + (5, [27.53003, 72.1106, 50.891026, 38.459953, 68.30715, 20.610682, 94.806274, 45.181377]), + (6, [77.73215, 64.42907, 71.50025, 43.85641, 94.42648, 50.04773, 65.12575, 68.58207]), + (7, [2.1537063, 82.667885, 16.171143, 71.126656, 5.335274, 40.286068, 11.943586, 3.69409]), + (8, [54.435013, 56.800594, 59.335514, 55.829235, 85.46627, 33.388138, 11.076194, 20.480877]), + (9, [76.197945, 60.623528, 84.229805, 31.652937, 71.82595, 48.04684, 71.29212, 30.282396]); + """ + + // CREATE INDEX + sql """ + CREATE INDEX idx_test_ann ON table_build_ann_index_on_mow(`embedding`) USING ANN PROPERTIES( + "index_type"="hnsw", + "metric_type"="l2_distance", + "dim"="8" + ); + """ + wait_for_latest_op_on_table_finish("table_build_ann_index_on_mow", timeout) + + // BUILD INDEX + sql "BUILD INDEX idx_test_ann ON table_build_ann_index_on_mow;" + wait_for_last_build_index_on_table_finish("table_build_ann_index_on_mow", timeout) } diff --git a/regression-test/suites/ann_index_p0/create_ann_index_test.groovy b/regression-test/suites/ann_index_p0/create_ann_index_test.groovy index 17f542de168475..3e32584a3786c6 100644 --- a/regression-test/suites/ann_index_p0/create_ann_index_test.groovy +++ b/regression-test/suites/ann_index_p0/create_ann_index_test.groovy @@ -39,6 +39,31 @@ suite("create_ann_index_test") { ); """ + sql "drop table if exists tbl_mor_without_ann" + sql """ + CREATE TABLE `tbl_mor_without_ann` ( + `id` int NOT NULL COMMENT "", + `embedding` array NOT NULL COMMENT "" + ) ENGINE=OLAP + UNIQUE KEY(`id`) COMMENT "OLAP" + DISTRIBUTED BY HASH(`id`) BUCKETS 2 + PROPERTIES ( + "replication_num" = "1", + "enable_unique_key_merge_on_write" = "false" + ); + """ + + test { + sql """ + CREATE INDEX idx_test_ann_on_mor ON tbl_mor_without_ann(`embedding`) USING ANN PROPERTIES( + "index_type"="hnsw", + "metric_type"="l2_distance", + "dim"="1" + ); + """ + exception "ANN index can only be built on table with DUP_KEYS or UNIQUE_KEYS with merge-on-write enabled" + } + // Test cases for creating tables with ANN indexes // 1. Case for nullable column @@ -299,26 +324,24 @@ suite("create_ann_index_test") { ); """ - sql "drop table if exists tbl_ann_unique_key" - test { - sql """ - CREATE TABLE tbl_ann_unique_key ( - id INT NOT NULL COMMENT "", - embedding ARRAY NOT NULL COMMENT "", - INDEX idx_test_ann (`embedding`) USING ANN PROPERTIES( - "index_type"="hnsw", - "metric_type"="inner_product", - "dim"="128" - ) - ) ENGINE=OLAP - UNIQUE KEY(id) COMMENT "OLAP" - DISTRIBUTED BY HASH(id) BUCKETS 2 - PROPERTIES ( - "replication_num" = "1" - ); - """ - exception "ANN index can only be used in DUP_KEYS table" - } + sql "drop table if exists tbl_ann_on_mow" + sql """ + CREATE TABLE tbl_ann_on_mow ( + id INT NOT NULL COMMENT "", + embedding ARRAY NOT NULL COMMENT "", + INDEX idx_test_ann (`embedding`) USING ANN PROPERTIES( + "index_type"="hnsw", + "metric_type"="inner_product", + "dim"="128" + ) + ) ENGINE=OLAP + UNIQUE KEY(id) COMMENT "OLAP" + DISTRIBUTED BY HASH(id) BUCKETS 2 + PROPERTIES ( + "replication_num" = "1", + "enable_unique_key_merge_on_write" = "true" + ); + """ sql "drop table if exists tbl_efconstruction" sql """ diff --git a/regression-test/suites/ann_index_p0/create_tbl_with_ann_index_test.groovy b/regression-test/suites/ann_index_p0/create_tbl_with_ann_index_test.groovy index 35725cd4b4688d..03c897d2760025 100644 --- a/regression-test/suites/ann_index_p0/create_tbl_with_ann_index_test.groovy +++ b/regression-test/suites/ann_index_p0/create_tbl_with_ann_index_test.groovy @@ -37,6 +37,28 @@ suite("create_tbl_with_ann_index_test") { """ } + sql "drop table if exists ann_tbl_mor" + test { + sql """ + CREATE TABLE ann_tbl_mor ( + id INT NOT NULL COMMENT "", + vec ARRAY NOT NULL COMMENT "", + INDEX ann_idx_mor (vec) USING ANN PROPERTIES( + "index_type" = "hnsw", + "metric_type" = "l2_distance", + "dim" = "128" + ) + ) ENGINE=OLAP + UNIQUE KEY(id) COMMENT "OLAP" + DISTRIBUTED BY HASH(id) BUCKETS 2 + PROPERTIES ( + "replication_num" = "1", + "enable_unique_key_merge_on_write" = "false" + ); + """ + exception "ANN index can only be used in DUP_KEYS table or UNIQUE_KEYS table with merge-on-write enabled" + } + sql "drop table if exists ann_tbl2" test { sql """