Java Token.setFlags方法代码示例

import org.apache.lucene.analysis.Token; //导入方法依赖的package包/类
protected void analyze(Collection<Token> result, String text, int offset, int flagsAttValue) throws IOException {
  TokenStream stream = analyzer.tokenStream("", text);
  // TODO: support custom attributes
  CharTermAttribute termAtt = stream.addAttribute(CharTermAttribute.class);
  TypeAttribute typeAtt = stream.addAttribute(TypeAttribute.class);
  PayloadAttribute payloadAtt = stream.addAttribute(PayloadAttribute.class);
  PositionIncrementAttribute posIncAtt = stream.addAttribute(PositionIncrementAttribute.class);
  OffsetAttribute offsetAtt = stream.addAttribute(OffsetAttribute.class);
  stream.reset();
  while (stream.incrementToken()) {      
    Token token = new Token();
    token.copyBuffer(termAtt.buffer(), 0, termAtt.length());
    token.setOffset(offset + offsetAtt.startOffset(), 
                    offset + offsetAtt.endOffset());
    token.setFlags(flagsAttValue); //overwriting any flags already set...
    token.setType(typeAtt.type());
    token.setPayload(payloadAtt.getPayload());
    token.setPositionIncrement(posIncAtt.getPositionIncrement());
    result.add(token);
  }
  stream.end();
  stream.close();
}
 

import org.apache.lucene.analysis.Token; //导入方法依赖的package包/类
private Token getNextPrefixInputToken(Token token) throws IOException {
  if (!prefix.incrementToken()) return null;
  token.copyBuffer(p_termAtt.buffer(), 0, p_termAtt.length());
  token.setPositionIncrement(p_posIncrAtt.getPositionIncrement());
  token.setFlags(p_flagsAtt.getFlags());
  token.setOffset(p_offsetAtt.startOffset(), p_offsetAtt.endOffset());
  token.setType(p_typeAtt.type());
  token.setPayload(p_payloadAtt.getPayload());
  return token;
}
 

import org.apache.lucene.analysis.Token; //导入方法依赖的package包/类
private Token getNextSuffixInputToken(Token token) throws IOException {
  if (!suffix.incrementToken()) return null;
  token.copyBuffer(termAtt.buffer(), 0, termAtt.length());
  token.setPositionIncrement(posIncrAtt.getPositionIncrement());
  token.setFlags(flagsAtt.getFlags());
  token.setOffset(offsetAtt.startOffset(), offsetAtt.endOffset());
  token.setType(typeAtt.type());
  token.setPayload(payloadAtt.getPayload());
  return token;
}
 

import org.apache.lucene.analysis.Token; //导入方法依赖的package包/类
@Override
public Collection<Token> convert(String origQuery) {
  Collection<Token> result = new HashSet<>();
  WhitespaceAnalyzer analyzer = new WhitespaceAnalyzer();
  
  TokenStream ts = null;
  try {
    ts = analyzer.tokenStream("", origQuery);
    // TODO: support custom attributes
    CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
    OffsetAttribute offsetAtt = ts.addAttribute(OffsetAttribute.class);
    TypeAttribute typeAtt = ts.addAttribute(TypeAttribute.class);
    FlagsAttribute flagsAtt = ts.addAttribute(FlagsAttribute.class);
    PayloadAttribute payloadAtt = ts.addAttribute(PayloadAttribute.class);
    PositionIncrementAttribute posIncAtt = ts.addAttribute(PositionIncrementAttribute.class);

    ts.reset();

    while (ts.incrementToken()) {
      Token tok = new Token();
      tok.copyBuffer(termAtt.buffer(), 0, termAtt.length());
      tok.setOffset(offsetAtt.startOffset(), offsetAtt.endOffset());
      tok.setFlags(flagsAtt.getFlags());
      tok.setPayload(payloadAtt.getPayload());
      tok.setPositionIncrement(posIncAtt.getPositionIncrement());
      tok.setType(typeAtt.type());
      result.add(tok);
    }
    ts.end();      
    return result;
  } catch (IOException e) {
    throw new RuntimeException(e);
  } finally {
    IOUtils.closeWhileHandlingException(ts);
  }
}
 

import org.apache.lucene.analysis.Token; //导入方法依赖的package包/类
/**
 * Converts the original query string to a collection of Lucene Tokens.
 * @param original the original query string
 * @return a Collection of Lucene Tokens
 */
@Override
public Collection<Token> convert(String original) {
  if (original == null) { // this can happen with q.alt = and no query
    return Collections.emptyList();
  }
  Collection<Token> result = new ArrayList<>();
  Matcher matcher = QUERY_REGEX.matcher(original);
  String nextWord = null;
  int nextStartIndex = 0;
  String lastBooleanOp = null;
  while (nextWord!=null || matcher.find()) {
    String word = null;
    int startIndex = 0;
    if(nextWord != null) {
      word = nextWord;
      startIndex = nextStartIndex;
      nextWord = null;
    } else {
      word = matcher.group(0);
      startIndex = matcher.start();
    }
    if(matcher.find()) {
      nextWord = matcher.group(0);
      nextStartIndex = matcher.start();
    }      
    if("AND".equals(word) || "OR".equals(word) || "NOT".equals(word)) {
      lastBooleanOp = word;        
      continue;
    }
    // treat "AND NOT" as "NOT"...
    if ("AND".equals(nextWord)
        && original.length() > nextStartIndex + 7
        && original.substring(nextStartIndex, nextStartIndex + 7).equals(
            "AND NOT")) {
      nextWord = "NOT";
    }
    
    int flagValue = 0;
    if (word.charAt(0) == '-'
        || (startIndex > 0 && original.charAt(startIndex - 1) == '-')) {
      flagValue = PROHIBITED_TERM_FLAG;
    } else if (word.charAt(0) == '+'
        || (startIndex > 0 && original.charAt(startIndex - 1) == '+')) {
      flagValue = REQUIRED_TERM_FLAG;
    //we don't know the default operator so just assume the first operator isn't new.
    } else if (nextWord != null
        && lastBooleanOp != null 
        && !nextWord.equals(lastBooleanOp)
        && ("AND".equals(nextWord) || "OR".equals(nextWord) || "NOT".equals(nextWord))) {
      flagValue = TERM_PRECEDES_NEW_BOOLEAN_OPERATOR_FLAG;
    //...unless the 1st boolean operator is a NOT, because only AND/OR can be default.
    } else if (nextWord != null
        && lastBooleanOp == null
        && !nextWord.equals(lastBooleanOp)
        && ("NOT".equals(nextWord))) {
      flagValue = TERM_PRECEDES_NEW_BOOLEAN_OPERATOR_FLAG;
    }
    try {
      analyze(result, word, startIndex, flagValue);
    } catch (IOException e) {
      // TODO: shouldn't we log something?
    }   
  }
  if(lastBooleanOp != null) {
    for(Token t : result) {
      int f = t.getFlags();
      t.setFlags(f |= QueryConverter.TERM_IN_BOOLEAN_QUERY_FLAG);
    }
  }
  return result;
}