當前位置: 首頁>>代碼示例>>Java>>正文

Java Token.length方法代碼示例

本文整理匯總了Java中org.apache.lucene.analysis.Token.length方法的典型用法代碼示例。如果您正苦於以下問題：Java Token.length方法的具體用法？Java Token.length怎麽用？Java Token.length使用的例子？那麽, 這裏精選的方法代碼示例或許可以為您提供幫助。您也可以進一步了解該方法所在類org.apache.lucene.analysis.Token的用法示例。


在下文中一共展示了Token.length方法的5個代碼示例，這些例子默認根據受歡迎程度排序。您可以為喜歡或者感覺有用的代碼點讚，您的評價將有助於係統推薦出更棒的Java代碼示例。

示例1: getLookupResults

import org.apache.lucene.analysis.Token; //導入方法依賴的package包/類
private List<LookupResult> getLookupResults(SpellingOptions options, Token currentToken) throws IOException {
    CharsRef scratch = new CharsRef();
    scratch.chars = currentToken.buffer();
    scratch.offset = 0;
    scratch.length = currentToken.length();
    boolean onlyMorePopular = (options.suggestMode == SuggestMode.SUGGEST_MORE_POPULAR) &&
            !(lookup instanceof WFSTCompletionLookup) &&
            !(lookup instanceof AnalyzingSuggester);

    List<LookupResult> suggestions = lookup.lookup(scratch, onlyMorePopular, options.count);
    if (suggestions == null || suggestions.size() == 0) {
        return null;
    }

    return suggestions;
}
開發者ID:DiceTechJobs，項目名稱:SolrPlugins，代碼行數:17，代碼來源:DiceMultipleCaseSuggester.java

示例2: getSuggestions

import org.apache.lucene.analysis.Token; //導入方法依賴的package包/類
@Override
public SpellingResult getSuggestions(SpellingOptions options) throws IOException {
  LOG.debug("getSuggestions: " + options.tokens);
  if (lookup == null) {
    LOG.info("Lookup is null - invoke spellchecker.build first");
    return EMPTY_RESULT;
  }
  SpellingResult res = new SpellingResult();
  CharsRef scratch = new CharsRef();
  for (Token t : options.tokens) {
    scratch.chars = t.buffer();
    scratch.offset = 0;
    scratch.length = t.length();
    boolean onlyMorePopular = (options.suggestMode == SuggestMode.SUGGEST_MORE_POPULAR) &&
      !(lookup instanceof WFSTCompletionLookup) &&
      !(lookup instanceof AnalyzingSuggester);
    List<LookupResult> suggestions = lookup.lookup(scratch, onlyMorePopular, options.count);
    if (suggestions == null) {
      continue;
    }
    if (options.suggestMode != SuggestMode.SUGGEST_MORE_POPULAR) {
      Collections.sort(suggestions);
    }
    for (LookupResult lr : suggestions) {
      res.add(t, lr.key.toString(), (int)lr.value);
    }
  }
  return res;
}
開發者ID:europeana，項目名稱:search，代碼行數:30，代碼來源:Suggester.java

示例3: toNamedList

import org.apache.lucene.analysis.Token; //導入方法依賴的package包/類
protected NamedList toNamedList(boolean shardRequest,
    SpellingResult spellingResult, String origQuery, boolean extendedResults,
    boolean collate, boolean correctlySpelled) {
  NamedList result = new NamedList();
  Map<Token,LinkedHashMap<String,Integer>> suggestions = spellingResult
      .getSuggestions();
  boolean hasFreqInfo = spellingResult.hasTokenFrequencyInfo();
  boolean hasSuggestions = false;
  boolean hasZeroFrequencyToken = false;
  for (Map.Entry<Token,LinkedHashMap<String,Integer>> entry : suggestions
      .entrySet()) {
    Token inputToken = entry.getKey();
    String tokenString = new String(inputToken.buffer(), 0, inputToken
        .length());
    Map<String,Integer> theSuggestions = new LinkedHashMap<>(
        entry.getValue());
    Iterator<String> sugIter = theSuggestions.keySet().iterator();
    while (sugIter.hasNext()) {
      String sug = sugIter.next();
      if (sug.equals(tokenString)) {
        sugIter.remove();
      }
    }
    if (theSuggestions.size() > 0) {
      hasSuggestions = true;
    }
    if (theSuggestions != null && (theSuggestions.size() > 0 || shardRequest)) {
      SimpleOrderedMap suggestionList = new SimpleOrderedMap();
      suggestionList.add("numFound", theSuggestions.size());
      suggestionList.add("startOffset", inputToken.startOffset());
      suggestionList.add("endOffset", inputToken.endOffset());
      
      // Logical structure of normal (non-extended) results:
      // "suggestion":["alt1","alt2"]
      //
      // Logical structure of the extended results:
      // "suggestion":[
      // {"word":"alt1","freq":7},
      // {"word":"alt2","freq":4}
      // ]
      if (extendedResults && hasFreqInfo) {
        suggestionList.add("origFreq", spellingResult
            .getTokenFrequency(inputToken));
        
        ArrayList<SimpleOrderedMap> sugs = new ArrayList<>();
        suggestionList.add("suggestion", sugs);
        for (Map.Entry<String,Integer> suggEntry : theSuggestions.entrySet()) {
          SimpleOrderedMap sugEntry = new SimpleOrderedMap();
          sugEntry.add("word", suggEntry.getKey());
          sugEntry.add("freq", suggEntry.getValue());
          sugs.add(sugEntry);
        }
      } else {
        suggestionList.add("suggestion", theSuggestions.keySet());
      }
      
      if (hasFreqInfo) {
        int tokenFrequency = spellingResult.getTokenFrequency(inputToken);
        if (tokenFrequency == 0) {
          hasZeroFrequencyToken = true;
        }
      }
      result.add(tokenString, suggestionList);
    }
  }
  
  if (extendedResults) {
    result.add("correctlySpelled", correctlySpelled);     
  }
  return result;
}
開發者ID:europeana，項目名稱:search，代碼行數:72，代碼來源:SpellCheckComponent.java

示例4: toNamedList

import org.apache.lucene.analysis.Token; //導入方法依賴的package包/類
protected NamedList toNamedList(boolean shardRequest,
                                SpellingResult spellingResult, String origQuery, boolean extendedResults,
                                boolean collate, boolean correctlySpelled) {
    NamedList result = new NamedList();
    Map<Token,LinkedHashMap<String,Integer>> suggestions = spellingResult
            .getSuggestions();
    boolean hasFreqInfo = spellingResult.hasTokenFrequencyInfo();
    boolean hasSuggestions = false;
    boolean hasZeroFrequencyToken = false;
    for (Map.Entry<Token,LinkedHashMap<String,Integer>> entry : suggestions
            .entrySet()) {
        Token inputToken = entry.getKey();
        String tokenString = new String(inputToken.buffer(), 0, inputToken
                .length());
        Map<String,Integer> theSuggestions = new LinkedHashMap<String,Integer>(
                entry.getValue());
        Iterator<String> sugIter = theSuggestions.keySet().iterator();
        while (sugIter.hasNext()) {
            String sug = sugIter.next();
            if (sug.equals(tokenString)) {
                sugIter.remove();
            }
        }
        if (theSuggestions.size() > 0) {
            hasSuggestions = true;
        }
        if (theSuggestions != null && (theSuggestions.size() > 0 || shardRequest)) {
            SimpleOrderedMap suggestionList = new SimpleOrderedMap();
            suggestionList.add("numFound", theSuggestions.size());
            suggestionList.add("startOffset", inputToken.startOffset());
            suggestionList.add("endOffset", inputToken.endOffset());

            // Logical structure of normal (non-extended) results:
            // "suggestion":["alt1","alt2"]
            //
            // Logical structure of the extended results:
            // "suggestion":[
            // {"word":"alt1","freq":7},
            // {"word":"alt2","freq":4}
            // ]
            if (extendedResults && hasFreqInfo) {
                suggestionList.add("origFreq", spellingResult
                        .getTokenFrequency(inputToken));

                ArrayList<SimpleOrderedMap> sugs = new ArrayList<SimpleOrderedMap>();
                suggestionList.add("suggestion", sugs);
                for (Map.Entry<String,Integer> suggEntry : theSuggestions.entrySet()) {
                    SimpleOrderedMap sugEntry = new SimpleOrderedMap();
                    sugEntry.add("word", suggEntry.getKey());
                    sugEntry.add("freq", suggEntry.getValue());
                    sugs.add(sugEntry);
                }
            } else {
                suggestionList.add("suggestion", theSuggestions.keySet());
            }

            if (hasFreqInfo) {
                int tokenFrequency = spellingResult.getTokenFrequency(inputToken);
                if (tokenFrequency == 0) {
                    hasZeroFrequencyToken = true;
                }
            }
            result.add(tokenString, suggestionList);
        }
    }

    if (extendedResults) {
        result.add("correctlySpelled", correctlySpelled);
    }
    return result;
}
開發者ID:DiceTechJobs，項目名稱:SolrPlugins，代碼行數:72，代碼來源:DiceSpellCheckComponent.java

示例5: getSuggestions

import org.apache.lucene.analysis.Token; //導入方法依賴的package包/類
@Override
public SpellingResult getSuggestions(SpellingOptions options) throws IOException {
    LOG.debug("getSuggestions: " + options.tokens);
    if (lookup == null) {
        LOG.info("Lookup is null - invoke spellchecker.build first");
        return EMPTY_RESULT;
    }
    SpellingResult res = new SpellingResult();
    CharsRef scratch = new CharsRef();

    for (Token currentToken : options.tokens) {
        scratch.chars = currentToken.buffer();
        scratch.offset = 0;
        scratch.length = currentToken.length();
        boolean onlyMorePopular = (options.suggestMode == SuggestMode.SUGGEST_MORE_POPULAR) &&
                !(lookup instanceof WFSTCompletionLookup) &&
                !(lookup instanceof AnalyzingSuggester);

        // get more than the requested suggestions as a lot get collapsed by the corrections
        List<LookupResult> suggestions = lookup.lookup(scratch, onlyMorePopular, options.count * 10);
        if (suggestions == null || suggestions.size() == 0) {
            continue;
        }

        if (options.suggestMode != SuggestMode.SUGGEST_MORE_POPULAR) {
            Collections.sort(suggestions);
        }

        final LinkedHashMap<String, Integer> lhm = new LinkedHashMap<String, Integer>();
        for (LookupResult lr : suggestions) {
            String suggestion = lr.key.toString();
            if(this.suggestionAnalyzer != null) {
                String correction = getAnalyzerResult(suggestion);
                // multiple could map to the same, so don't repeat suggestions
                if(!isStringNullOrEmpty(correction)){
                    if(lhm.containsKey(correction)){
                        lhm.put(correction, lhm.get(correction) + (int) lr.value);
                    }
                    else {
                        lhm.put(correction, (int) lr.value);
                    }
                }
            }
            else {
                lhm.put(suggestion, (int) lr.value);
            }

            if(lhm.size() >= options.count){
                break;
            }
        }

        // sort by new doc frequency
        Map<String, Integer> orderedMap = null;
        if (options.suggestMode != SuggestMode.SUGGEST_MORE_POPULAR){
            // retain the sort order from above
            orderedMap = lhm;
        }
        else {
            orderedMap = new TreeMap<String, Integer>(new Comparator<String>() {
                @Override
                public int compare(String s1, String s2) {
                    return lhm.get(s2).compareTo(lhm.get(s1));
                }
            });
            orderedMap.putAll(lhm);
        }

        for(Map.Entry<String, Integer> entry: orderedMap.entrySet()){
            res.add(currentToken, entry.getKey(), entry.getValue());
        }

    }
    return res;
}
開發者ID:DiceTechJobs，項目名稱:SolrPlugins，代碼行數:76，代碼來源:DiceSuggester.java


注：本文中的org.apache.lucene.analysis.Token.length方法示例由純淨天空整理自Github/MSDocs等開源代碼及文檔管理平台，相關代碼片段篩選自各路編程大神貢獻的開源項目，源碼版權歸原作者所有，傳播和使用請參考對應項目的License；未經允許，請勿轉載。