apache · serhiy-bzhezytskyy · Aug 25, 2026
diff --git a/solr/core/src/java/org/apache/solr/handler/component/SpellCheckComponent.java b/solr/core/src/java/org/apache/solr/handler/component/SpellCheckComponent.java
@@ -20,7 +20,6 @@
 import java.lang.invoke.MethodHandles;
 import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -29,15 +28,11 @@
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.function.Supplier;
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.core.WhitespaceAnalyzer;
 import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
-import org.apache.lucene.analysis.tokenattributes.FlagsAttribute;
-import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
-import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
-import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.ExitableDirectoryReader;
 import org.apache.lucene.search.Query;
@@ -69,10 +64,10 @@
 import org.apache.solr.spelling.SolrSpellChecker;
 import org.apache.solr.spelling.SpellCheckCollation;
 import org.apache.solr.spelling.SpellCheckCollator;
+import org.apache.solr.spelling.SpellCheckToken;
 import org.apache.solr.spelling.SpellingOptions;
 import org.apache.solr.spelling.SpellingQueryConverter;
 import org.apache.solr.spelling.SpellingResult;
-import org.apache.solr.spelling.Token;
 import org.apache.solr.util.SolrResponseUtil;
 import org.apache.solr.util.plugin.SolrCoreAware;
 import org.slf4j.Logger;
@@ -143,20 +138,33 @@ public void process(ResponseBuilder rb) throws IOException {
 
     SolrSpellChecker spellChecker = getSpellChecker(params);
     if (spellChecker != null) {
-      Collection<Token> tokens;
       String q = params.get(SPELLCHECK_Q);
+      final String convertedQ;
+      Supplier<TokenStream> tokenStreamSupplier;
       if (q != null) {
         // we have a spell check param, tokenize it with the query analyzer applicable for this
         // spellchecker
-        tokens = getTokens(q, spellChecker.getQueryAnalyzer());
+        convertedQ = q;
+        tokenStreamSupplier = () -> getTokens(convertedQ, spellChecker.getQueryAnalyzer());
       } else {
         q = rb.getQueryString();
         if (q == null) {
           q = params.get(CommonParams.Q);
         }
-        tokens = queryConverter.convert(q);
+        convertedQ = q;
+        tokenStreamSupplier = () -> queryConverter.convert(convertedQ);
       }
-      if (tokens != null && tokens.isEmpty() == false) {
+      // peek for at least one token, using our own throwaway stream instance. Must fully drain
+      // before end()/close() -- TokenStream forbids abandoning mid-INCREMENT.
+      boolean hasTokens = false;
+      try (TokenStream peek = tokenStreamSupplier.get()) {
+        peek.reset();
+        while (peek.incrementToken()) {
+          hasTokens = true;
+        }
+        peek.end();
+      }
+      if (hasTokens) {
         int count = params.getInt(SPELLCHECK_COUNT, 1);
         boolean onlyMorePopular =
             params.getBool(SPELLCHECK_ONLY_MORE_POPULAR, DEFAULT_ONLY_MORE_POPULAR);
@@ -198,7 +206,7 @@ public void process(ResponseBuilder rb) throws IOException {
           }
           SpellingOptions options =
               new SpellingOptions(
-                  tokens,
+                  tokenStreamSupplier,
                   reader,
                   count,
                   alternativeTermCount,
@@ -230,7 +238,7 @@ public void process(ResponseBuilder rb) throws IOException {
               params, spellingResult, rb, q, response, spellChecker.isSuggestionsMayOverlap());
         }
         if (shardRequest) {
-          addOriginalTermsToResponse(response, tokens);
+          addOriginalTermsToResponse(response, tokenStreamSupplier.get());
         }
 
         rb.rsp.add("spellcheck", response);
@@ -346,12 +354,16 @@ protected void addCollationsToResponse(
     response.add("collations", collationList);
   }
 
-  private void addOriginalTermsToResponse(
-      NamedList<Object> response, Collection<Token> originalTerms) {
-    List<String> originalTermStr = new ArrayList<String>();
-    for (Token t : originalTerms) {
-      originalTermStr.add(t.toString());
+  private void addOriginalTermsToResponse(NamedList<Object> response, TokenStream originalTerms)
+      throws IOException {
+    List<String> originalTermStr = new ArrayList<>();
+    originalTerms.reset();
+    CharTermAttribute termAtt = originalTerms.addAttribute(CharTermAttribute.class);
+    while (originalTerms.incrementToken()) {
+      originalTermStr.add(termAtt.toString());
     }
+    originalTerms.end();
+    originalTerms.close();
     response.add("originalTerms", originalTermStr);
   }
 
@@ -580,32 +592,9 @@ private void collectShardCollations(
     }
   }
 
-  private Collection<Token> getTokens(String q, Analyzer analyzer) throws IOException {
-    Collection<Token> result = new ArrayList<>();
+  private TokenStream getTokens(String q, Analyzer analyzer) {
     assert analyzer != null;
-    try (TokenStream ts = analyzer.tokenStream("", q)) {
-      ts.reset();
-      // TODO: support custom attributes
-      CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
-      OffsetAttribute offsetAtt = ts.addAttribute(OffsetAttribute.class);
-      TypeAttribute typeAtt = ts.addAttribute(TypeAttribute.class);
-      FlagsAttribute flagsAtt = ts.addAttribute(FlagsAttribute.class);
-      PayloadAttribute payloadAtt = ts.addAttribute(PayloadAttribute.class);
-      PositionIncrementAttribute posIncAtt = ts.addAttribute(PositionIncrementAttribute.class);
-
-      while (ts.incrementToken()) {
-        Token token = new Token();
-        token.copyBuffer(termAtt.buffer(), 0, termAtt.length());
-        token.setOffset(offsetAtt.startOffset(), offsetAtt.endOffset());
-        token.setType(typeAtt.type());
-        token.setFlags(flagsAtt.getFlags());
-        token.setPayload(payloadAtt.getPayload());
-        token.setPositionIncrement(posIncAtt.getPositionIncrement());
-        result.add(token);
-      }
-      ts.end();
-      return result;
-    }
+    return analyzer.tokenStream("", q);
   }
 
   protected SolrSpellChecker getSpellChecker(SolrParams params) {
@@ -658,13 +647,15 @@ protected NamedList<Object> toNamedList(
       String origQuery,
       boolean extendedResults) {
     NamedList<Object> result = new NamedList<>();
-    Map<Token, LinkedHashMap<String, Integer>> suggestions = spellingResult.getSuggestions();
+    Map<SpellCheckToken, LinkedHashMap<String, Integer>> suggestions =
+        spellingResult.getSuggestions();
     boolean hasFreqInfo = spellingResult.hasTokenFrequencyInfo();
     boolean hasSuggestions = false;
     boolean hasZeroFrequencyToken = false;
-    for (Map.Entry<Token, LinkedHashMap<String, Integer>> entry : suggestions.entrySet()) {
-      Token inputToken = entry.getKey();
-      String tokenString = new String(inputToken.buffer(), 0, inputToken.length());
+    for (Map.Entry<SpellCheckToken, LinkedHashMap<String, Integer>> entry :
+        suggestions.entrySet()) {
+      SpellCheckToken inputToken = entry.getKey();
+      String tokenString = inputToken.text();
       Map<String, Integer> theSuggestions = new LinkedHashMap<>(entry.getValue());
       theSuggestions.keySet().removeIf(sug -> sug.equals(tokenString));
       if (theSuggestions.size() > 0) {

diff --git a/solr/core/src/java/org/apache/solr/spelling/AbstractLuceneSpellChecker.java b/solr/core/src/java/org/apache/solr/spelling/AbstractLuceneSpellChecker.java
@@ -21,6 +21,13 @@
 import java.util.Arrays;
 import java.util.Comparator;
 import java.util.List;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.analysis.tokenattributes.FlagsAttribute;
+import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
+import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
+import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
+import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.spell.Dictionary;
@@ -132,19 +139,36 @@ public String init(NamedList<?> config, SolrCore core) {
 
   @Override
   public SpellingResult getSuggestions(SpellingOptions options) throws IOException {
-    SpellingResult result = new SpellingResult(options.tokens);
+    SpellingResult result = new SpellingResult();
     IndexReader reader = determineReader(options.reader);
     Term term = field != null ? new Term(field, "") : null;
     float theAccuracy =
         (options.accuracy == Float.MIN_VALUE) ? spellChecker.getAccuracy() : options.accuracy;
 
     int count = Math.max(options.count, AbstractLuceneSpellChecker.DEFAULT_SUGGESTION_COUNT);
-    for (Token token : options.tokens) {
-      if (token.length() == 0) {
+    TokenStream stream = options.tokenStreamSupplier.get();
+    stream.reset();
+    CharTermAttribute termAtt = stream.addAttribute(CharTermAttribute.class);
+    OffsetAttribute offsetAtt = stream.addAttribute(OffsetAttribute.class);
+    TypeAttribute typeAtt = stream.addAttribute(TypeAttribute.class);
+    PositionIncrementAttribute posIncAtt = stream.addAttribute(PositionIncrementAttribute.class);
+    FlagsAttribute flagsAtt = stream.addAttribute(FlagsAttribute.class);
+    PayloadAttribute payloadAtt = stream.addAttribute(PayloadAttribute.class);
+    while (stream.incrementToken()) {
+      String tokenText = termAtt.toString();
+      SpellCheckToken token =
+          new SpellCheckToken(
+              tokenText,
+              offsetAtt.startOffset(),
+              offsetAtt.endOffset(),
+              typeAtt.type(),
+              posIncAtt.getPositionIncrement(),
+              flagsAtt.getFlags(),
+              payloadAtt.getPayload());
+      if (tokenText.isEmpty()) {
         result.add(token, List.of());
         continue;
       }
-      String tokenText = new String(token.buffer(), 0, token.length());
       term = new Term(field, tokenText);
       int docFreq = 0;
       if (reader != null) {
@@ -209,6 +233,8 @@ public SpellingResult getSuggestions(SpellingOptions options) throws IOException
         }
       }
     }
+    stream.end();
+    stream.close();
     return result;
   }
 

diff --git a/solr/core/src/java/org/apache/solr/spelling/ConjunctionSolrSpellChecker.java b/solr/core/src/java/org/apache/solr/spelling/ConjunctionSolrSpellChecker.java
@@ -130,13 +130,14 @@ public SpellingResult mergeSuggestions(
   // TODO: This just interleaves the results.  In the future, we might want to let users give each
   // checker its own weight and use that in combination to score & frequency to sort the results ?
   private SpellingResult mergeCheckers(SpellingResult[] results, int numSug) {
-    Map<Token, Integer> combinedTokenFrequency = new HashMap<>();
-    Map<Token, List<LinkedHashMap<String, Integer>>> allSuggestions = new LinkedHashMap<>();
+    Map<SpellCheckToken, Integer> combinedTokenFrequency = new HashMap<>();
+    Map<SpellCheckToken, List<LinkedHashMap<String, Integer>>> allSuggestions =
+        new LinkedHashMap<>();
     for (SpellingResult result : results) {
       if (result.getTokenFrequency() != null) {
         combinedTokenFrequency.putAll(result.getTokenFrequency());
       }
-      for (Map.Entry<Token, LinkedHashMap<String, Integer>> entry :
+      for (Map.Entry<SpellCheckToken, LinkedHashMap<String, Integer>> entry :
           result.getSuggestions().entrySet()) {
         List<LinkedHashMap<String, Integer>> allForThisToken = allSuggestions.get(entry.getKey());
         if (allForThisToken == null) {
@@ -147,8 +148,9 @@ private SpellingResult mergeCheckers(SpellingResult[] results, int numSug) {
       }
     }
     SpellingResult combinedResult = new SpellingResult();
-    for (Map.Entry<Token, List<LinkedHashMap<String, Integer>>> entry : allSuggestions.entrySet()) {
-      Token original = entry.getKey();
+    for (Map.Entry<SpellCheckToken, List<LinkedHashMap<String, Integer>>> entry :
+        allSuggestions.entrySet()) {
+      SpellCheckToken original = entry.getKey();
       List<Iterator<Map.Entry<String, Integer>>> corrIters =
           new ArrayList<>(entry.getValue().size());
       for (LinkedHashMap<String, Integer> corrections : entry.getValue()) {

diff --git a/solr/core/src/java/org/apache/solr/spelling/DirectSolrSpellChecker.java b/solr/core/src/java/org/apache/solr/spelling/DirectSolrSpellChecker.java
@@ -21,6 +21,13 @@
 import java.util.ArrayList;
 import java.util.Comparator;
 import java.util.List;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.analysis.tokenattributes.FlagsAttribute;
+import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
+import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
+import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
+import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.spell.DirectSpellChecker;
 import org.apache.lucene.search.spell.StringDistance;
@@ -183,18 +190,33 @@ public void build(SolrCore core, SolrIndexSearcher searcher) throws IOException
 
   @Override
   public SpellingResult getSuggestions(SpellingOptions options) throws IOException {
-    log.debug("getSuggestions: {}", options.tokens);
-
     SpellingResult result = new SpellingResult();
     float accuracy =
         (options.accuracy == Float.MIN_VALUE) ? checker.getAccuracy() : options.accuracy;
 
-    for (Token token : options.tokens) {
-      if (token.length() == 0) {
+    TokenStream stream = options.tokenStreamSupplier.get();
+    stream.reset();
+    CharTermAttribute termAtt = stream.addAttribute(CharTermAttribute.class);
+    OffsetAttribute offsetAtt = stream.addAttribute(OffsetAttribute.class);
+    TypeAttribute typeAtt = stream.addAttribute(TypeAttribute.class);
+    PositionIncrementAttribute posIncAtt = stream.addAttribute(PositionIncrementAttribute.class);
+    FlagsAttribute flagsAtt = stream.addAttribute(FlagsAttribute.class);
+    PayloadAttribute payloadAtt = stream.addAttribute(PayloadAttribute.class);
+    while (stream.incrementToken()) {
+      String tokenText = termAtt.toString();
+      SpellCheckToken token =
+          new SpellCheckToken(
+              tokenText,
+              offsetAtt.startOffset(),
+              offsetAtt.endOffset(),
+              typeAtt.type(),
+              posIncAtt.getPositionIncrement(),
+              flagsAtt.getFlags(),
+              payloadAtt.getPayload());
+      if (tokenText.isEmpty()) {
         result.add(token, List.of());
         continue;
       }
-      String tokenText = token.toString();
       Term term = new Term(field, tokenText);
       int freq = options.reader.docFreq(term);
       int count =
@@ -234,6 +256,8 @@ public SpellingResult getSuggestions(SpellingOptions options) throws IOException
         }
       }
     }
+    stream.end();
+    stream.close();
     return result;
   }
 

diff --git a/solr/core/src/java/org/apache/solr/spelling/PossibilityIterator.java b/solr/core/src/java/org/apache/solr/spelling/PossibilityIterator.java
@@ -55,13 +55,14 @@ private PossibilityIterator() {
    * Possible Correction".
    */
   public PossibilityIterator(
-      Map<Token, LinkedHashMap<String, Integer>> suggestions,
+      Map<SpellCheckToken, LinkedHashMap<String, Integer>> suggestions,
       int maximumRequiredSuggestions,
       int maxEvaluations,
       boolean overlap) {
     this.suggestionsMayOverlap = overlap;
-    for (Map.Entry<Token, LinkedHashMap<String, Integer>> entry : suggestions.entrySet()) {
-      Token token = entry.getKey();
+    for (Map.Entry<SpellCheckToken, LinkedHashMap<String, Integer>> entry :
+        suggestions.entrySet()) {
+      SpellCheckToken token = entry.getKey();
       if (entry.getValue().size() == 0) {
         continue;
       }

diff --git a/solr/core/src/java/org/apache/solr/spelling/QueryConverter.java b/solr/core/src/java/org/apache/solr/spelling/QueryConverter.java
@@ -16,8 +16,8 @@
  */
 package org.apache.solr.spelling;
 
-import java.util.Collection;
 import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.TokenStream;
 import org.apache.solr.util.plugin.NamedListInitializedPlugin;
 
 /**
@@ -57,10 +57,13 @@ public abstract class QueryConverter implements NamedListInitializedPlugin {
   public static final int TERM_IN_BOOLEAN_QUERY_FLAG = 131072;
 
   /**
-   * Returns the Collection of {@link Token}s for the query. Offsets on the Token should correspond
-   * to the correct offset in the origQuery
+   * Returns a fresh {@link TokenStream} over the query's terms. Offsets should correspond to the
+   * correct offset in the origQuery. The caller owns the returned stream's lifecycle (reset, then
+   * an incrementToken loop, then end, then close); this method may be called more than once for the
+   * same query text, e.g. via {@link SpellingOptions#tokenStreamSupplier}, since a {@link
+   * TokenStream} is single-use.
    */
-  public abstract Collection<Token> convert(String original);
+  public abstract TokenStream convert(String original);
 
   /** Set the analyzer to use. Must be set before any calls to convert. */
   public void setAnalyzer(Analyzer analyzer) {