Java Token.toString方法代码示例

import org.apache.lucene.analysis.Token; //导入方法依赖的package包/类
@Override
public SpellingResult getSuggestions(SpellingOptions options)
    throws IOException {
  LOG.debug("getSuggestions: " + options.tokens);
      
  SpellingResult result = new SpellingResult();
  float accuracy = (options.accuracy == Float.MIN_VALUE) ? checker.getAccuracy() : options.accuracy;
  
  for (Token token : options.tokens) {
    String tokenText = token.toString();
    Term term = new Term(field, tokenText);
    int freq = options.reader.docFreq(term);
    int count = (options.alternativeTermCount > 0 && freq > 0) ? options.alternativeTermCount: options.count;
    SuggestWord[] suggestions = checker.suggestSimilar(term, count,options.reader, options.suggestMode, accuracy);
    result.addFrequency(token, freq);
          
    // If considering alternatives to "correctly-spelled" terms, then add the
    // original as a viable suggestion.
    if (options.alternativeTermCount > 0 && freq > 0) {
      boolean foundOriginal = false;
      SuggestWord[] suggestionsWithOrig = new SuggestWord[suggestions.length + 1];
      for (int i = 0; i < suggestions.length; i++) {
        if (suggestions[i].string.equals(tokenText)) {
          foundOriginal = true;
          break;
        }
        suggestionsWithOrig[i + 1] = suggestions[i];
      }
      if (!foundOriginal) {
        SuggestWord orig = new SuggestWord();
        orig.freq = freq;
        orig.string = tokenText;
        suggestionsWithOrig[0] = orig;
        suggestions = suggestionsWithOrig;
      }
    }      
    if(suggestions.length==0 && freq==0) {
      List<String> empty = Collections.emptyList();
      result.add(token, empty);
    } else {        
      for (SuggestWord suggestion : suggestions) {
        result.add(token, suggestion.string, suggestion.freq);
      }
    }
  }
  return result;
}
 

import org.apache.lucene.analysis.Token; //导入方法依赖的package包/类
@Override
public SpellingResult getSuggestions(SpellingOptions options)
        throws IOException {

    LOG.debug("getSuggestions: " + options.tokens);
    // load the typos file if not loaded

    SpellingResult result = new SpellingResult();
    float accuracy = (options.accuracy == Float.MIN_VALUE) ? checker.getAccuracy() : options.accuracy;

    for (Token token : options.tokens) {
        String tokenText = token.toString();
        Term term = new Term(field, tokenText);
        int freq = options.reader.docFreq(term);
        int count = (options.alternativeTermCount > 0  && freq > 0) ? options.alternativeTermCount: options.count;
        SuggestWord[] suggestions = checker.suggestSimilar(term, count,options.reader, options.suggestMode, accuracy);
        result.addFrequency(token, freq);

        // Dice functionality: Allow also loading of a list of spelling corrections to apply in addition
        // to the standard functionality. This allows us to configure common typos to correct that may exceed the
        // max edit distance used by solr
        if(this.typosLoaded){
            String normTokenText = normalize(tokenText);
            String match = this.mapTypos.get(normTokenText);
            if(match != null){
                int matchFreq = options.reader.docFreq(new Term(field, match));
                // only ever suggest values that are in the index and more frequent
                // than the original word
                if(matchFreq > 0 && matchFreq > freq) {
                    result.add(token, match, matchFreq);
                }
            }
        }

        // If considering alternatives to "correctly-spelled" terms, then add the
        // original as a viable suggestion.
        if (options.alternativeTermCount > 0 && freq > 0) {
            boolean foundOriginal = false;
            SuggestWord[] suggestionsWithOrig = new SuggestWord[suggestions.length + 1];
            for (int i = 0; i < suggestions.length; i++) {
                if (suggestions[i].string.equals(tokenText)) {
                    foundOriginal = true;
                    break;
                }
                suggestionsWithOrig[i + 1] = suggestions[i];
            }
            if (!foundOriginal) {
                SuggestWord orig = new SuggestWord();
                orig.freq = freq;
                orig.string = tokenText;
                suggestionsWithOrig[0] = orig;
                suggestions = suggestionsWithOrig;
            }
        }
        if(suggestions.length==0 && freq==0) {
            List<String> empty = Collections.emptyList();
            result.add(token, empty);
        } else {
            for (SuggestWord suggestion : suggestions) {
                result.add(token, suggestion.string, suggestion.freq);
            }
        }
    }
    return result;
}
 

import org.apache.lucene.analysis.Token; //导入方法依赖的package包/类
@Override
public SpellingResult getSuggestions(SpellingOptions options) throws IOException {
    LOG.debug("getSuggestions: " + options.tokens);
    if (lookup == null) {
        LOG.info("Lookup is null - invoke spellchecker.build first");
        return EMPTY_RESULT;
    }

    SpellingResult res = new SpellingResult();
    for (Token currentToken : options.tokens) {
        String tokenText = currentToken.toString();

        // we need to ensure that we combine matches for different cases, and take the most common
        // where multiple case versions exist
        final Hashtable<String, LookupResult> htSuggestions = new Hashtable<String, LookupResult>();
        final Hashtable<String, Integer> htSuggestionCounts = new Hashtable<String, Integer>();

        for(String sToken: generateCaseVariants(tokenText)){

            Token newToken = newToken(currentToken, sToken);
            List<LookupResult> tmpSuggestions = getLookupResults(options, newToken);
            if(tmpSuggestions != null){
                for(LookupResult lu: tmpSuggestions) {
                    final String key = lu.key.toString().toLowerCase();
                    LookupResult existing = htSuggestions.get(key);
                    if(existing != null) {
                        // replace if more frequent
                        if (lu.value > existing.value) {
                            htSuggestions.put(key, lu);
                        }
                        htSuggestionCounts.put(key, htSuggestionCounts.get(key) + (int)lu.value);
                    }
                    else{
                        htSuggestions.put(key, lu);
                        htSuggestionCounts.put(key, (int)lu.value);
                    }
                }
            }
        }

        List<String> suggestions = new ArrayList<String>(htSuggestions.keySet());
        if (options.suggestMode != SuggestMode.SUGGEST_MORE_POPULAR) {
            Collections.sort(suggestions);
        }
        else{
            Collections.sort(suggestions, new Comparator<String>() {
                public int compare(String sug1, String sug2) {
                    int sug1Count = htSuggestionCounts.get(sug1);
                    int sug2Count = htSuggestionCounts.get(sug2);
                    return sug2Count - sug1Count;
                }
            });
        }

        for (String match : suggestions) {
            LookupResult lr = htSuggestions.get(match);
            res.add(currentToken, lr.key.toString(), (int)lr.value);
        }

    }
    return res;
}