From 4b0b29c442503c18e6a4ef6124529f81a2089d95 Mon Sep 17 00:00:00 2001 From: Pascal Seitz Date: Thu, 22 Oct 2026 12:24:56 +0200 Subject: [PATCH 1/2] Add datadog_v2 tokenizer with dot-separated parts --- docs/configuration/index-config.md | 4 + .../src/tokenizers/datadog_v2.rs | 202 ++++++++++++++++++ quickwit/quickwit-query/src/tokenizers/mod.rs | 10 + 3 files changed, 216 insertions(+) create mode 100644 quickwit/quickwit-query/src/tokenizers/datadog_v2.rs diff --git a/docs/configuration/index-config.md b/docs/configuration/index-config.md index 876311e96cb..ddc88ba259c 100644 --- a/docs/configuration/index-config.md +++ b/docs/configuration/index-config.md @@ -149,6 +149,10 @@ fast: | `chinese_compatible` | Chop between each CJK character in addition to what `default` does. Should be used with `record: position` to be able to properly search | | `lowercase` | Applies a lowercase transformation on the text. It does not tokenize the text. | +`datadog` (also named `unicode_segmenter`) splits text at Unicode word boundaries, lowercases tokens, and applies the 255-byte token-length limit. It preserves internal punctuation in words such as `can't`, `example.com`, and `32.3`. + +`datadog_v2` additionally emits the dot-separated parts of each token, preserving the original token. For example, `Foo.Bar.Baz` produces `foo.bar.baz`, `foo`, `bar`, and `baz`. Tokens containing only numbers and dots, such as `192.168.1.1` and `32.3`, stay whole. Parts occupy consecutive positions, and the original token spans those positions. Lowercasing and the token-length limit apply independently to the original and its parts. Select it with `tokenizer: datadog_v2` in a text field mapping. + ##### Description of available normalizers | Normalizer | Description | diff --git a/quickwit/quickwit-query/src/tokenizers/datadog_v2.rs b/quickwit/quickwit-query/src/tokenizers/datadog_v2.rs new file mode 100644 index 00000000000..59b10c0ef09 --- /dev/null +++ b/quickwit/quickwit-query/src/tokenizers/datadog_v2.rs @@ -0,0 +1,202 @@ +// Copyright 2021-Present Datadog, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use tantivy::tokenizer::{Token, TokenStream, Tokenizer}; +use unicode_segmentation::UnicodeSegmentation; + +/// Unicode word tokenization with additional dot-separated parts. +/// +/// The original token is emitted first and spans the positions occupied by its +/// parts. Tokens containing only numbers and dots (e.g. IP addresses and decimals) +/// stay whole. Only ASCII dots are separators. +#[derive(Clone, Default)] +pub struct DatadogV2Tokenizer; + +pub struct DatadogV2TokenStream<'a> { + words: unicode_segmentation::UnicodeWordIndices<'a>, + parts: Option>, + part_offset: usize, + position: usize, + token: Token, +} + +impl Tokenizer for DatadogV2Tokenizer { + type TokenStream<'a> = DatadogV2TokenStream<'a>; + + fn token_stream<'a>(&'a mut self, text: &'a str) -> Self::TokenStream<'a> { + DatadogV2TokenStream { + words: text.unicode_word_indices(), + parts: None, + part_offset: 0, + position: 0, + token: Token::default(), + } + } +} + +impl DatadogV2TokenStream<'_> { + fn set_token(&mut self, text: &str, offset: usize, position_length: usize) { + self.token.text.clear(); + self.token.text.push_str(text); + self.token.offset_from = offset; + self.token.offset_to = offset + text.len(); + self.token.position = self.position; + self.token.position_length = position_length; + } +} + +impl TokenStream for DatadogV2TokenStream<'_> { + fn advance(&mut self) -> bool { + if let Some(part) = self.parts.as_mut().and_then(Iterator::next) { + self.set_token(part, self.part_offset, 1); + self.part_offset += part.len() + 1; + self.position += 1; + return true; + } + self.parts = None; + + let Some((offset, word)) = self.words.next() else { + return false; + }; + let split = word.contains('.') + && !word + .chars() + .all(|character| character == '.' || character.is_numeric()); + if split { + // Unicode word boundaries exclude leading, trailing, and consecutive + // dots, so every part is nonempty. The first part shares the original's + // position, and subsequent parts occupy consecutive positions. + let parts = word.split('.'); + self.set_token(word, offset, parts.clone().count()); + self.parts = Some(parts); + self.part_offset = offset; + } else { + self.set_token(word, offset, 1); + self.position += 1; + } + true + } + + fn token(&self) -> &Token { + &self.token + } + + fn token_mut(&mut self) -> &mut Token { + &mut self.token + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::tokenizers::create_default_quickwit_tokenizer_manager; + + fn tokens(text: &str) -> Vec { + let mut tokenizer = DatadogV2Tokenizer; + let mut stream = tokenizer.token_stream(text); + let mut tokens = Vec::new(); + while stream.advance() { + tokens.push(stream.token().clone()); + } + tokens + } + + #[test] + fn test_datadog_v2_parts_offsets_and_positions() { + let actual: Vec<_> = tokens("é Foo.Bär.baz end") + .into_iter() + .map(|token| { + ( + token.text, + token.offset_from, + token.offset_to, + token.position, + token.position_length, + ) + }) + .collect(); + let expected = [ + ("é", 0, 2, 0, 1), + ("Foo.Bär.baz", 3, 15, 1, 3), + ("Foo", 3, 6, 1, 1), + ("Bär", 7, 11, 2, 1), + ("baz", 12, 15, 3, 1), + ("end", 16, 19, 4, 1), + ] + .map(|(text, start, end, position, length)| { + (text.to_string(), start, end, position, length) + }); + assert_eq!(actual, expected); + } + + #[test] + fn test_datadog_v2_numeric_tokens() { + let actual: Vec<_> = tokens("192.168.1.1 32.3 １２.３ 1.2.3 abc.def 42") + .into_iter() + .map(|token| token.text) + .collect(); + assert_eq!( + actual, + [ + "192.168.1.1", + "32.3", + "１２.３", + "1.2.3", + "abc.def", + "abc", + "def", + "42" + ] + ); + } + + #[test] + fn test_datadog_v2_word_boundaries() { + for (text, expected) in [ + ("", vec![]), + ("...", vec![]), + (".foo..bar.", vec!["foo", "bar"]), + ( + "can't foo.bar baz.qux", + vec!["can't", "foo.bar", "foo", "bar", "baz.qux", "baz", "qux"], + ), + ("v1.2.3", vec!["v1.2.3", "v1", "2", "3"]), + ] { + let actual: Vec<_> = tokens(text).into_iter().map(|token| token.text).collect(); + assert_eq!(actual, expected, "{text}"); + } + } + + #[test] + fn test_datadog_v2_analyzer() { + let manager = create_default_quickwit_tokenizer_manager(); + let long_part = "a".repeat(256); + let text = format!("Foo.BAR 192.168.1.1 {long_part}.OK"); + for (name, expected) in [ + ("datadog", vec!["foo.bar", "192.168.1.1"]), + ( + "datadog_v2", + vec!["foo.bar", "foo", "bar", "192.168.1.1", "ok"], + ), + ] { + let mut analyzer = manager.get_tokenizer(name).unwrap(); + let mut stream = analyzer.token_stream(&text); + let mut actual = Vec::new(); + while stream.advance() { + actual.push(stream.token().text.clone()); + } + assert_eq!(actual, expected, "{name}"); + } + } +} diff --git a/quickwit/quickwit-query/src/tokenizers/mod.rs b/quickwit/quickwit-query/src/tokenizers/mod.rs index 4e2ee53b15d..3d7d10f19b6 100644 --- a/quickwit/quickwit-query/src/tokenizers/mod.rs +++ b/quickwit/quickwit-query/src/tokenizers/mod.rs @@ -14,6 +14,7 @@ mod chinese_compatible; mod code_tokenizer; +mod datadog_v2; mod tokenizer_manager; mod unicode_segmenter_tokenizer; @@ -26,6 +27,7 @@ use tantivy::tokenizer::{ use self::chinese_compatible::ChineseTokenizer; pub use self::code_tokenizer::CodeTokenizer; +use self::datadog_v2::DatadogV2Tokenizer; pub use self::tokenizer_manager::{RAW_TOKENIZER_NAME, TokenizerManager}; pub use self::unicode_segmenter_tokenizer::UnicodeSegmenterTokenizer; @@ -92,6 +94,14 @@ pub fn create_default_quickwit_tokenizer_manager() -> TokenizerManager { true, ); tokenizer_manager.register("datadog", unicode_segmenter_tokenizer, true); + tokenizer_manager.register( + "datadog_v2", + TextAnalyzer::builder(DatadogV2Tokenizer) + .filter(LowerCaser) + .filter(RemoveLongFilter::limit(DEFAULT_REMOVE_TOKEN_LENGTH)) + .build(), + true, + ); tokenizer_manager } From 438a121f7352c444af635d481a52a93d9ec3cc54 Mon Sep 17 00:00:00 2001 From: Pascal Seitz Date: Thu, 22 Oct 2026 13:32:55 +0200 Subject: [PATCH 2/2] Fix dotted prefix queries with datadog_v2 --- docs/configuration/index-config.md | 6 +- .../src/query_ast/full_text_query.rs | 51 ++++++++---- .../src/query_ast/phrase_prefix_query.rs | 27 +++++++ .../quickwit-query/src/query_ast/utils.rs | 4 +- .../src/tokenizers/datadog_v2.rs | 77 ++---------------- .../tokenizers/unicode_segmenter_tokenizer.rs | 80 ++++++++++++++----- 6 files changed, 135 insertions(+), 110 deletions(-) diff --git a/docs/configuration/index-config.md b/docs/configuration/index-config.md index ddc88ba259c..cc04d7218ba 100644 --- a/docs/configuration/index-config.md +++ b/docs/configuration/index-config.md @@ -148,10 +148,8 @@ fast: | `whitespace` | Chops the text on according to whitespace only. Doesn't remove long tokens or converts to lowercase. | | `chinese_compatible` | Chop between each CJK character in addition to what `default` does. Should be used with `record: position` to be able to properly search | | `lowercase` | Applies a lowercase transformation on the text. It does not tokenize the text. | - -`datadog` (also named `unicode_segmenter`) splits text at Unicode word boundaries, lowercases tokens, and applies the 255-byte token-length limit. It preserves internal punctuation in words such as `can't`, `example.com`, and `32.3`. - -`datadog_v2` additionally emits the dot-separated parts of each token, preserving the original token. For example, `Foo.Bar.Baz` produces `foo.bar.baz`, `foo`, `bar`, and `baz`. Tokens containing only numbers and dots, such as `192.168.1.1` and `32.3`, stay whole. Parts occupy consecutive positions, and the original token spans those positions. Lowercasing and the token-length limit apply independently to the original and its parts. Select it with `tokenizer: datadog_v2` in a text field mapping. +| `datadog`, `unicode_segmenter` | Splits at Unicode word boundaries, lowercases tokens, and applies the 255-byte token-length limit. | +| `datadog_v2` | Like `datadog`, plus dot-separated parts: `Foo.Bar` → `foo.bar`, `foo`, `bar`. Numeric dotted tokens such as `192.168.1.1` and `32.3` stay whole. | ##### Description of available normalizers diff --git a/quickwit/quickwit-query/src/query_ast/full_text_query.rs b/quickwit/quickwit-query/src/query_ast/full_text_query.rs index 83c4ce5d7b3..a0f731b5ea4 100644 --- a/quickwit/quickwit-query/src/query_ast/full_text_query.rs +++ b/quickwit/quickwit-query/src/query_ast/full_text_query.rs @@ -65,6 +65,7 @@ impl FullTextParams { text: &str, json_options: &JsonObjectOptions, tokenizer_manager: &TokenizerManager, + is_prefix: bool, ) -> anyhow::Result> { let text_indexing_options = json_options .get_text_indexing_options() @@ -72,14 +73,12 @@ impl FullTextParams { let mut text_analyzer: TextAnalyzer = self.text_analyzer(text_indexing_options, tokenizer_manager)?; let mut token_stream = text_analyzer.token_stream(text); - let mut tokens = Vec::new(); - token_stream.process(&mut |token| { + Ok(collect_terms(&mut token_stream, is_prefix, |text| { let mut term = Term::from_field_json_path(field, json_path, json_options.is_expand_dots_enabled()); - term.append_type_and_str(&token.text); - tokens.push((token.position, term)); - }); - Ok(tokens) + term.append_type_and_str(text); + term + })) } pub(crate) fn tokenize_text_into_terms( @@ -88,16 +87,14 @@ impl FullTextParams { text: &str, text_field_indexing: &TextFieldIndexing, tokenizer_manager: &TokenizerManager, + is_prefix: bool, ) -> anyhow::Result> { let mut text_analyzer: TextAnalyzer = self.text_analyzer(text_field_indexing, tokenizer_manager)?; let mut token_stream = text_analyzer.token_stream(text); - let mut tokens = Vec::new(); - token_stream.process(&mut |token| { - let term: Term = Term::from_field_text(field, &token.text); - tokens.push((token.position, term)); - }); - Ok(tokens) + Ok(collect_terms(&mut token_stream, is_prefix, |text| { + Term::from_field_text(field, text) + })) } pub(crate) fn make_query( @@ -108,7 +105,7 @@ impl FullTextParams { if terms.is_empty() { return Ok(self.zero_terms_query.into()); } - if terms.len() == 1 { + if terms.len() == 1 && !matches!(self.mode, FullTextMode::BoolPrefix { .. }) { let term = terms.pop().unwrap().1; return Ok(TantivyTermQuery::new(term, IndexRecordOption::WithFreqs).into()); } @@ -163,6 +160,29 @@ impl FullTextParams { } } +/// Keep the final original token as the prefix, without requiring its parts as +/// exact terms. Tokenizers such as datadog_v2 emit the original first, with a +/// position_length spanning its subsequent parts. Earlier words keep all terms. +fn collect_terms( + token_stream: &mut dyn TokenStream, + is_prefix: bool, + make_term: impl Fn(&str) -> Term, +) -> Vec<(usize, Term)> { + let mut terms = Vec::new(); + let mut prefix_span: Option<(usize, usize)> = None; + token_stream.process(&mut |token| { + if is_prefix && !matches!(prefix_span, Some((_, end)) if token.position < end) { + prefix_span = (token.position_length > 1) + .then_some((terms.len(), token.position + token.position_length)); + } + terms.push((token.position, make_term(&token.text))); + }); + if let Some((index, _)) = prefix_span { + terms.truncate(index + 1); + } + terms +} + fn is_zero(val: &u32) -> bool { *val == 0u32 } @@ -209,8 +229,7 @@ impl From for FullTextMode { /// The tokens might be transformed into a phrase queries, /// into a disjunction, or into a conjunction. /// -/// If after tokenization, a single term is emitted, it will naturally be -/// produce a tantivy TermQuery. +/// A single token produces a term query, or a prefix query in BoolPrefix mode. /// /// If no terms is emitted, it will produce a query that match all or no documents, /// depending on `full_text_params.zero_terms_query`. @@ -275,6 +294,7 @@ impl FullTextQuery { &self.text, text_field_indexing, tokenizer_manager, + true, ) .ok()?; let (_pos, term) = terms.pop()?; @@ -289,6 +309,7 @@ impl FullTextQuery { &self.text, json_options, tokenizer_manager, + true, ) .ok()?; let (_pos, term) = terms.pop()?; diff --git a/quickwit/quickwit-query/src/query_ast/phrase_prefix_query.rs b/quickwit/quickwit-query/src/query_ast/phrase_prefix_query.rs index ef6658e647c..8fb31a73988 100644 --- a/quickwit/quickwit-query/src/query_ast/phrase_prefix_query.rs +++ b/quickwit/quickwit-query/src/query_ast/phrase_prefix_query.rs @@ -68,6 +68,7 @@ impl PhrasePrefixQuery { &self.phrase, text_field_indexing, tokenizer_manager, + true, )?; (text_field_indexing, term_positions) } @@ -85,6 +86,7 @@ impl PhrasePrefixQuery { &self.phrase, json_options, tokenizer_manager, + true, )?; (text_field_indexing, term_positions) } @@ -340,6 +342,31 @@ mod tests { assert_eq!(count_matches(&query, &index), 3); } + #[test] + fn test_datadog_v2_dotted_prefix() { + let mut schema_builder = TantivySchema::builder(); + let body = schema_builder.add_text_field( + "body", + TextOptions::default().set_indexing_options( + TextFieldIndexing::default() + .set_tokenizer("datadog_v2") + .set_index_option(tantivy::schema::IndexRecordOption::WithFreqsAndPositions), + ), + ); + let mut index = Index::create_in_ram(schema_builder.build()); + index.set_tokenizers( + crate::create_default_quickwit_tokenizer_manager() + .tantivy_manager() + .clone(), + ); + let mut writer = index.writer_with_num_threads(1, 20_000_000).unwrap(); + writer.add_document(doc!(body => "foo.bar baz")).unwrap(); + writer.commit().unwrap(); + + let query = phrase_prefix_query("body", "foo.ba", 50); + assert_eq!(count_matches(&query, &index), 1); + } + #[test] fn test_empty_tokenization_matches_none_by_default() { let mut schema_builder = TantivySchema::builder(); diff --git a/quickwit/quickwit-query/src/query_ast/utils.rs b/quickwit/quickwit-query/src/query_ast/utils.rs index 6c930916dc9..6c3bdc39332 100644 --- a/quickwit/quickwit-query/src/query_ast/utils.rs +++ b/quickwit/quickwit-query/src/query_ast/utils.rs @@ -23,7 +23,7 @@ use tantivy::schema::{ use crate::InvalidQuery; use crate::MatchAllOrNone::MatchNone as TantivyEmptyQuery; use crate::json_literal::InterpretUserInput; -use crate::query_ast::full_text_query::FullTextParams; +use crate::query_ast::full_text_query::{FullTextMode, FullTextParams}; use crate::query_ast::tantivy_query_ast::{TantivyBoolQuery, TantivyQueryAst}; use crate::tokenizers::{RAW_TOKENIZER_NAME, TokenizerManager}; @@ -168,6 +168,7 @@ fn compute_query_with_field( value, text_field_indexing, tokenizer_manager, + matches!(full_text_params.mode, FullTextMode::BoolPrefix { .. }), )?; full_text_params.make_query(terms, text_field_indexing.index_option()) } @@ -219,6 +220,7 @@ fn compute_tantivy_ast_query_for_json( text, json_options, tokenizer_manager, + matches!(full_text_params.mode, FullTextMode::BoolPrefix { .. }), )?; let index_record_option = json_options .get_text_indexing_options() diff --git a/quickwit/quickwit-query/src/tokenizers/datadog_v2.rs b/quickwit/quickwit-query/src/tokenizers/datadog_v2.rs index 59b10c0ef09..f52becc8f25 100644 --- a/quickwit/quickwit-query/src/tokenizers/datadog_v2.rs +++ b/quickwit/quickwit-query/src/tokenizers/datadog_v2.rs @@ -12,8 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -use tantivy::tokenizer::{Token, TokenStream, Tokenizer}; -use unicode_segmentation::UnicodeSegmentation; +use tantivy::tokenizer::Tokenizer; + +use super::unicode_segmenter_tokenizer::UnicodeSegmenterTokenStream; /// Unicode word tokenization with additional dot-separated parts. /// @@ -23,82 +24,18 @@ use unicode_segmentation::UnicodeSegmentation; #[derive(Clone, Default)] pub struct DatadogV2Tokenizer; -pub struct DatadogV2TokenStream<'a> { - words: unicode_segmentation::UnicodeWordIndices<'a>, - parts: Option>, - part_offset: usize, - position: usize, - token: Token, -} - impl Tokenizer for DatadogV2Tokenizer { - type TokenStream<'a> = DatadogV2TokenStream<'a>; + type TokenStream<'a> = UnicodeSegmenterTokenStream<'a>; fn token_stream<'a>(&'a mut self, text: &'a str) -> Self::TokenStream<'a> { - DatadogV2TokenStream { - words: text.unicode_word_indices(), - parts: None, - part_offset: 0, - position: 0, - token: Token::default(), - } - } -} - -impl DatadogV2TokenStream<'_> { - fn set_token(&mut self, text: &str, offset: usize, position_length: usize) { - self.token.text.clear(); - self.token.text.push_str(text); - self.token.offset_from = offset; - self.token.offset_to = offset + text.len(); - self.token.position = self.position; - self.token.position_length = position_length; - } -} - -impl TokenStream for DatadogV2TokenStream<'_> { - fn advance(&mut self) -> bool { - if let Some(part) = self.parts.as_mut().and_then(Iterator::next) { - self.set_token(part, self.part_offset, 1); - self.part_offset += part.len() + 1; - self.position += 1; - return true; - } - self.parts = None; - - let Some((offset, word)) = self.words.next() else { - return false; - }; - let split = word.contains('.') - && !word - .chars() - .all(|character| character == '.' || character.is_numeric()); - if split { - // Unicode word boundaries exclude leading, trailing, and consecutive - // dots, so every part is nonempty. The first part shares the original's - // position, and subsequent parts occupy consecutive positions. - let parts = word.split('.'); - self.set_token(word, offset, parts.clone().count()); - self.parts = Some(parts); - self.part_offset = offset; - } else { - self.set_token(word, offset, 1); - self.position += 1; - } - true - } - - fn token(&self) -> &Token { - &self.token - } - - fn token_mut(&mut self) -> &mut Token { - &mut self.token + UnicodeSegmenterTokenStream::new(text, true) } } #[cfg(test)] mod tests { + use tantivy::tokenizer::{Token, TokenStream}; + use super::*; use crate::tokenizers::create_default_quickwit_tokenizer_manager; diff --git a/quickwit/quickwit-query/src/tokenizers/unicode_segmenter_tokenizer.rs b/quickwit/quickwit-query/src/tokenizers/unicode_segmenter_tokenizer.rs index cac935b1cc4..cb55749f31f 100644 --- a/quickwit/quickwit-query/src/tokenizers/unicode_segmenter_tokenizer.rs +++ b/quickwit/quickwit-query/src/tokenizers/unicode_segmenter_tokenizer.rs @@ -24,27 +24,67 @@ use unicode_segmentation::UnicodeSegmentation; pub struct UnicodeSegmenterTokenizer; pub struct UnicodeSegmenterTokenStream<'a> { - iter: unicode_segmentation::UnicodeWordIndices<'a>, - token: Token, + words: unicode_segmentation::UnicodeWordIndices<'a>, + emit_dot_parts: bool, + parts: Option>, + part_offset: usize, position: usize, + token: Token, +} + +impl<'a> UnicodeSegmenterTokenStream<'a> { + pub(super) fn new(text: &'a str, emit_dot_parts: bool) -> Self { + Self { + words: text.unicode_word_indices(), + emit_dot_parts, + parts: None, + part_offset: 0, + position: 0, + token: Token::default(), + } + } + + fn set_token(&mut self, text: &str, offset: usize, position_length: usize) { + self.token.text.clear(); + self.token.text.push_str(text); + self.token.offset_from = offset; + self.token.offset_to = offset + text.len(); + self.token.position = self.position; + self.token.position_length = position_length; + } } -impl<'a> TokenStream for UnicodeSegmenterTokenStream<'a> { +impl TokenStream for UnicodeSegmenterTokenStream<'_> { fn advance(&mut self) -> bool { - if let Some((start, word)) = self.iter.next() { - let pos = self.position; - let end = start + word.len(); - let token = self.token_mut(); - token.text.clear(); - token.text.push_str(word); - token.offset_from = start; - token.offset_to = end; - token.position = pos; + if let Some(part) = self.parts.as_mut().and_then(Iterator::next) { + self.set_token(part, self.part_offset, 1); + self.part_offset += part.len() + 1; self.position += 1; - true + return true; + } + self.parts = None; + + let Some((offset, word)) = self.words.next() else { + return false; + }; + let split = self.emit_dot_parts + && word.contains('.') + && !word + .chars() + .all(|character| character == '.' || character.is_numeric()); + if split { + // Unicode word boundaries exclude leading, trailing, and consecutive + // dots, so every part is nonempty. The first part shares the original's + // position, and subsequent parts occupy consecutive positions. + let parts = word.split('.'); + self.set_token(word, offset, parts.clone().count()); + self.parts = Some(parts); + self.part_offset = offset; } else { - false + self.set_token(word, offset, 1); + self.position += 1; } + true } fn token(&self) -> &Token { @@ -60,11 +100,7 @@ impl Tokenizer for UnicodeSegmenterTokenizer { type TokenStream<'a> = UnicodeSegmenterTokenStream<'a>; fn token_stream<'a>(&'a mut self, text: &'a str) -> Self::TokenStream<'a> { - UnicodeSegmenterTokenStream { - iter: text.unicode_word_indices(), - token: Token::default(), - position: 0, - } + UnicodeSegmenterTokenStream::new(text, false) } } @@ -74,13 +110,15 @@ mod tests { #[test] fn test_unicode_segmenter_tokenizer() { - let text = r#"The quick ("brown") fox can't jump 32.3 feet, right?"#; + let text = r#"The quick ("brown") fox can't jump 32.3 feet, right? Foo.Bar end"#; let mut tokenizer = UnicodeSegmenterTokenizer; let mut stream = tokenizer.token_stream(text); let mut tokens = Vec::new(); while stream.advance() { let token = stream.token().clone(); + assert_eq!(token.position, tokens.len()); + assert_eq!(token.position_length, 1); tokens.push((token.offset_from, token.offset_to, token.text)); } @@ -94,6 +132,8 @@ mod tests { (35, 39, "32.3"), (40, 44, "feet"), (46, 51, "right"), + (53, 60, "Foo.Bar"), + (61, 64, "end"), ]; assert_eq!( tokens,