Add datadog_v2 tokenizer with dot-separated parts

docs/configuration/index-config.md

@@ -148,6 +148,8 @@ fast:
 | `whitespace`  | Chops the text on according to whitespace only. Doesn't remove long tokens or converts to lowercase. |
 | `chinese_compatible` |  Chop between each CJK character in addition to what `default` does. Should be used with `record: position` to be able to properly search |
 | `lowercase`   | Applies a lowercase transformation on the text. It does not tokenize the text. |
+| `datadog`, `unicode_segmenter` | Splits at Unicode word boundaries, lowercases tokens, and applies the 255-byte token-length limit. |
+| `datadog_v2` | Like `datadog`, plus dot-separated parts: `Foo.Bar` → `foo.bar`, `foo`, `bar`. Numeric dotted tokens such as `192.168.1.1` and `32.3` stay whole. |
 
 ##### Description of available normalizers
 

quickwit/quickwit-query/src/query_ast/full_text_query.rs

@@ -65,21 +65,20 @@ impl FullTextParams {
         text: &str,
         json_options: &JsonObjectOptions,
         tokenizer_manager: &TokenizerManager,
+        is_prefix: bool,
     ) -> anyhow::Result<Vec<(usize, Term)>> {
         let text_indexing_options = json_options
             .get_text_indexing_options()
             .with_context(|| format!("Json field text `{json_path}` is not indexed"))?;
         let mut text_analyzer: TextAnalyzer =
             self.text_analyzer(text_indexing_options, tokenizer_manager)?;
         let mut token_stream = text_analyzer.token_stream(text);
-        let mut tokens = Vec::new();
-        token_stream.process(&mut |token| {
+        Ok(collect_terms(&mut token_stream, is_prefix, |text| {
             let mut term =
                 Term::from_field_json_path(field, json_path, json_options.is_expand_dots_enabled());
-            term.append_type_and_str(&token.text);
-            tokens.push((token.position, term));
-        });
-        Ok(tokens)
+            term.append_type_and_str(text);
+            term
+        }))
     }
 
     pub(crate) fn tokenize_text_into_terms(
@@ -88,16 +87,14 @@ impl FullTextParams {
         text: &str,
         text_field_indexing: &TextFieldIndexing,
         tokenizer_manager: &TokenizerManager,
+        is_prefix: bool,
     ) -> anyhow::Result<Vec<(usize, Term)>> {
         let mut text_analyzer: TextAnalyzer =
             self.text_analyzer(text_field_indexing, tokenizer_manager)?;
         let mut token_stream = text_analyzer.token_stream(text);
-        let mut tokens = Vec::new();
-        token_stream.process(&mut |token| {
-            let term: Term = Term::from_field_text(field, &token.text);
-            tokens.push((token.position, term));
-        });
-        Ok(tokens)
+        Ok(collect_terms(&mut token_stream, is_prefix, |text| {
+            Term::from_field_text(field, text)
+        }))
     }
 
     pub(crate) fn make_query(
@@ -108,7 +105,7 @@ impl FullTextParams {
         if terms.is_empty() {
             return Ok(self.zero_terms_query.into());
         }
-        if terms.len() == 1 {
+        if terms.len() == 1 && !matches!(self.mode, FullTextMode::BoolPrefix { .. }) {
             let term = terms.pop().unwrap().1;
             return Ok(TantivyTermQuery::new(term, IndexRecordOption::WithFreqs).into());
         }
@@ -163,6 +160,29 @@ impl FullTextParams {
     }
 }
 
+/// Keep the final original token as the prefix, without requiring its parts as
+/// exact terms. Tokenizers such as datadog_v2 emit the original first, with a
+/// position_length spanning its subsequent parts. Earlier words keep all terms.
+fn collect_terms(
+    token_stream: &mut dyn TokenStream,
+    is_prefix: bool,
+    make_term: impl Fn(&str) -> Term,
+) -> Vec<(usize, Term)> {
+    let mut terms = Vec::new();
+    let mut prefix_span: Option<(usize, usize)> = None;
+    token_stream.process(&mut |token| {
+        if is_prefix && !matches!(prefix_span, Some((_, end)) if token.position < end) {
+            prefix_span = (token.position_length > 1)
+                .then_some((terms.len(), token.position + token.position_length));
+        }
+        terms.push((token.position, make_term(&token.text)));
+    });
+    if let Some((index, _)) = prefix_span {
+        terms.truncate(index + 1);
+    }
+    terms
+}
+
 fn is_zero(val: &u32) -> bool {
     *val == 0u32
 }
@@ -209,8 +229,7 @@ impl From<BooleanOperand> for FullTextMode {
 /// The tokens might be transformed into a phrase queries,
 /// into a disjunction, or into a conjunction.
 ///
-/// If after tokenization, a single term is emitted, it will naturally be
-/// produce a tantivy TermQuery.
+/// A single token produces a term query, or a prefix query in BoolPrefix mode.
 ///
 /// If no terms is emitted, it will produce a query that match all or no documents,
 /// depending on `full_text_params.zero_terms_query`.
@@ -275,6 +294,7 @@ impl FullTextQuery {
                         &self.text,
                         text_field_indexing,
                         tokenizer_manager,
+                        true,
                     )
                     .ok()?;
                 let (_pos, term) = terms.pop()?;
@@ -289,6 +309,7 @@ impl FullTextQuery {
                         &self.text,
                         json_options,
                         tokenizer_manager,
+                        true,
                     )
                     .ok()?;
                 let (_pos, term) = terms.pop()?;

quickwit/quickwit-query/src/query_ast/phrase_prefix_query.rs

@@ -68,6 +68,7 @@ impl PhrasePrefixQuery {
                     &self.phrase,
                     text_field_indexing,
                     tokenizer_manager,
+                    true,
                 )?;
                 (text_field_indexing, term_positions)
             }
@@ -85,6 +86,7 @@ impl PhrasePrefixQuery {
                     &self.phrase,
                     json_options,
                     tokenizer_manager,
+                    true,
                 )?;
                 (text_field_indexing, term_positions)
             }
@@ -340,6 +342,31 @@ mod tests {
         assert_eq!(count_matches(&query, &index), 3);
     }
 
+    #[test]
+    fn test_datadog_v2_dotted_prefix() {
+        let mut schema_builder = TantivySchema::builder();
+        let body = schema_builder.add_text_field(
+            "body",
+            TextOptions::default().set_indexing_options(
+                TextFieldIndexing::default()
+                    .set_tokenizer("datadog_v2")
+                    .set_index_option(tantivy::schema::IndexRecordOption::WithFreqsAndPositions),
+            ),
+        );
+        let mut index = Index::create_in_ram(schema_builder.build());
+        index.set_tokenizers(
+            crate::create_default_quickwit_tokenizer_manager()
+                .tantivy_manager()
+                .clone(),
+        );
+        let mut writer = index.writer_with_num_threads(1, 20_000_000).unwrap();
+        writer.add_document(doc!(body => "foo.bar baz")).unwrap();
+        writer.commit().unwrap();
+
+        let query = phrase_prefix_query("body", "foo.ba", 50);
+        assert_eq!(count_matches(&query, &index), 1);
+    }
+
     #[test]
     fn test_empty_tokenization_matches_none_by_default() {
         let mut schema_builder = TantivySchema::builder();

quickwit/quickwit-query/src/query_ast/utils.rs

@@ -23,7 +23,7 @@ use tantivy::schema::{
 use crate::InvalidQuery;
 use crate::MatchAllOrNone::MatchNone as TantivyEmptyQuery;
 use crate::json_literal::InterpretUserInput;
-use crate::query_ast::full_text_query::FullTextParams;
+use crate::query_ast::full_text_query::{FullTextMode, FullTextParams};
 use crate::query_ast::tantivy_query_ast::{TantivyBoolQuery, TantivyQueryAst};
 use crate::tokenizers::{RAW_TOKENIZER_NAME, TokenizerManager};
 
@@ -168,6 +168,7 @@ fn compute_query_with_field(
                 value,
                 text_field_indexing,
                 tokenizer_manager,
+                matches!(full_text_params.mode, FullTextMode::BoolPrefix { .. }),
             )?;
             full_text_params.make_query(terms, text_field_indexing.index_option())
         }
@@ -219,6 +220,7 @@ fn compute_tantivy_ast_query_for_json(
         text,
         json_options,
         tokenizer_manager,
+        matches!(full_text_params.mode, FullTextMode::BoolPrefix { .. }),
     )?;
     let index_record_option = json_options
         .get_text_indexing_options()

quickwit/quickwit-query/src/tokenizers/datadog_v2.rs

@@ -0,0 +1,139 @@
+// Copyright 2021-Present Datadog, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use tantivy::tokenizer::Tokenizer;
+
+use super::unicode_segmenter_tokenizer::UnicodeSegmenterTokenStream;
+
+/// Unicode word tokenization with additional dot-separated parts.
+///
+/// The original token is emitted first and spans the positions occupied by its
+/// parts. Tokens containing only numbers and dots (e.g. IP addresses and decimals)
+/// stay whole. Only ASCII dots are separators.
+#[derive(Clone, Default)]
+pub struct DatadogV2Tokenizer;
+
+impl Tokenizer for DatadogV2Tokenizer {
+    type TokenStream<'a> = UnicodeSegmenterTokenStream<'a>;
+
+    fn token_stream<'a>(&'a mut self, text: &'a str) -> Self::TokenStream<'a> {
+        UnicodeSegmenterTokenStream::new(text, true)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use tantivy::tokenizer::{Token, TokenStream};
+
+    use super::*;
+    use crate::tokenizers::create_default_quickwit_tokenizer_manager;
+
+    fn tokens(text: &str) -> Vec<Token> {
+        let mut tokenizer = DatadogV2Tokenizer;
+        let mut stream = tokenizer.token_stream(text);
+        let mut tokens = Vec::new();
+        while stream.advance() {
+            tokens.push(stream.token().clone());
+        }
+        tokens
+    }
+
+    #[test]
+    fn test_datadog_v2_parts_offsets_and_positions() {
+        let actual: Vec<_> = tokens("é Foo.Bär.baz end")
+            .into_iter()
+            .map(|token| {
+                (
+                    token.text,
+                    token.offset_from,
+                    token.offset_to,
+                    token.position,
+                    token.position_length,
+                )
+            })
+            .collect();
+        let expected = [
+            ("é", 0, 2, 0, 1),
+            ("Foo.Bär.baz", 3, 15, 1, 3),
+            ("Foo", 3, 6, 1, 1),
+            ("Bär", 7, 11, 2, 1),
+            ("baz", 12, 15, 3, 1),
+            ("end", 16, 19, 4, 1),
+        ]
+        .map(|(text, start, end, position, length)| {
+            (text.to_string(), start, end, position, length)
+        });
+        assert_eq!(actual, expected);
+    }
+
+    #[test]
+    fn test_datadog_v2_numeric_tokens() {
+        let actual: Vec<_> = tokens("192.168.1.1 32.3 １２.３ 1.2.3 abc.def 42")
+            .into_iter()
+            .map(|token| token.text)
+            .collect();
+        assert_eq!(
+            actual,
+            [
+                "192.168.1.1",
+                "32.3",
+                "１２.３",
+                "1.2.3",
+                "abc.def",
+                "abc",
+                "def",
+                "42"
+            ]
+        );
+    }
+
+    #[test]
+    fn test_datadog_v2_word_boundaries() {
+        for (text, expected) in [
+            ("", vec![]),
+            ("...", vec![]),
+            (".foo..bar.", vec!["foo", "bar"]),
+            (
+                "can't foo.bar baz.qux",
+                vec!["can't", "foo.bar", "foo", "bar", "baz.qux", "baz", "qux"],
+            ),
+            ("v1.2.3", vec!["v1.2.3", "v1", "2", "3"]),
+        ] {
+            let actual: Vec<_> = tokens(text).into_iter().map(|token| token.text).collect();
+            assert_eq!(actual, expected, "{text}");
+        }
+    }
+
+    #[test]
+    fn test_datadog_v2_analyzer() {
+        let manager = create_default_quickwit_tokenizer_manager();
+        let long_part = "a".repeat(256);
+        let text = format!("Foo.BAR 192.168.1.1 {long_part}.OK");
+        for (name, expected) in [
+            ("datadog", vec!["foo.bar", "192.168.1.1"]),
+            (
+                "datadog_v2",
+                vec!["foo.bar", "foo", "bar", "192.168.1.1", "ok"],
+            ),
+        ] {
+            let mut analyzer = manager.get_tokenizer(name).unwrap();
+            let mut stream = analyzer.token_stream(&text);
+            let mut actual = Vec::new();
+            while stream.advance() {
+                actual.push(stream.token().text.clone());
+            }
+            assert_eq!(actual, expected, "{name}");
+        }
+    }
+}

quickwit/quickwit-query/src/tokenizers/mod.rs

@@ -14,6 +14,7 @@
 
 mod chinese_compatible;
 mod code_tokenizer;
+mod datadog_v2;
 mod tokenizer_manager;
 mod unicode_segmenter_tokenizer;
 
@@ -26,6 +27,7 @@ use tantivy::tokenizer::{
 
 use self::chinese_compatible::ChineseTokenizer;
 pub use self::code_tokenizer::CodeTokenizer;
+use self::datadog_v2::DatadogV2Tokenizer;
 pub use self::tokenizer_manager::{RAW_TOKENIZER_NAME, TokenizerManager};
 pub use self::unicode_segmenter_tokenizer::UnicodeSegmenterTokenizer;
 
@@ -92,6 +94,14 @@ pub fn create_default_quickwit_tokenizer_manager() -> TokenizerManager {
         true,
     );
     tokenizer_manager.register("datadog", unicode_segmenter_tokenizer, true);
+    tokenizer_manager.register(
+        "datadog_v2",
+        TextAnalyzer::builder(DatadogV2Tokenizer)
+            .filter(LowerCaser)
+            .filter(RemoveLongFilter::limit(DEFAULT_REMOVE_TOKEN_LENGTH))
+            .build(),
+        true,
+    );
     tokenizer_manager
 }