Java AttributeSource.State方法代码示例

import org.apache.lucene.util.AttributeSource; //导入方法依赖的package包/类
@Override
public boolean incrementToken() throws IOException {
  if (input.incrementToken()) {
    // capture state lazily - maybe no SinkFilter accepts this state
    AttributeSource.State state = null;
    for (WeakReference<SinkTokenStream> ref : sinks) {
      final SinkTokenStream sink = ref.get();
      if (sink != null) {
        if (sink.accept(this)) {
          if (state == null) {
            state = this.captureState();
          }
          sink.addState(state);
        }
      }
    }
    return true;
  }
  
  return false;
}
 

import org.apache.lucene.util.AttributeSource; //导入方法依赖的package包/类
@Override
public final boolean incrementToken() throws IOException {
  if (cache == null) {
    // fill cache lazily
    cache = new LinkedList<AttributeSource.State>();
    fillCache();
    iterator = cache.iterator();
  }
  
  if (!iterator.hasNext()) {
    // the cache is exhausted, return false
    return false;
  }
  // Since the TokenFilter can be reset, the tokens need to be preserved as immutable.
  restoreState(iterator.next());
  return true;
}
 

import org.apache.lucene.util.AttributeSource; //导入方法依赖的package包/类
@Override
public final void end() throws IOException {
  super.end();
  AttributeSource.State finalState = captureState();
  for (WeakReference<SinkTokenStream> ref : sinks) {
    final SinkTokenStream sink = ref.get();
    if (sink != null) {
      sink.setFinalState(finalState);
    }
  }
}
 

import org.apache.lucene.util.AttributeSource; //导入方法依赖的package包/类
@Override
public final boolean incrementToken() throws IOException {
  if (tokens != null && tokens.hasNext()){
    AttributeSource.State state = tokens.next();
    restoreState(state);
    return true;
  }
  clearAttributes();
  int tokenType = scanner.getNextToken();

  if (tokenType == WikipediaTokenizerImpl.YYEOF) {
    return false;
  }
  String type = WikipediaTokenizerImpl.TOKEN_TYPES[tokenType];
  if (tokenOutput == TOKENS_ONLY || untokenizedTypes.contains(type) == false){
    setupToken();
  } else if (tokenOutput == UNTOKENIZED_ONLY && untokenizedTypes.contains(type) == true){
    collapseTokens(tokenType);

  }
  else if (tokenOutput == BOTH){
    //collapse into a single token, add it to tokens AND output the individual tokens
    //output the untokenized Token first
    collapseAndSaveTokens(tokenType, type);
  }
  int posinc = scanner.getPositionIncrement();
  if (first && posinc == 0) {
    posinc = 1; // don't emit posinc=0 for the first token!
  }
  posIncrAtt.setPositionIncrement(posinc);
  typeAtt.setType(type);
  first = false;
  return true;
}
 

import org.apache.lucene.util.AttributeSource; //导入方法依赖的package包/类
private void collapseAndSaveTokens(int tokenType, String type) throws IOException {
  //collapse
  StringBuilder buffer = new StringBuilder(32);
  int numAdded = scanner.setText(buffer);
  //TODO: how to know how much whitespace to add
  int theStart = scanner.yychar();
  int lastPos = theStart + numAdded;
  int tmpTokType;
  int numSeen = 0;
  List<AttributeSource.State> tmp = new ArrayList<>();
  setupSavedToken(0, type);
  tmp.add(captureState());
  //while we can get a token and that token is the same type and we have not transitioned to a new wiki-item of the same type
  while ((tmpTokType = scanner.getNextToken()) != WikipediaTokenizerImpl.YYEOF && tmpTokType == tokenType && scanner.getNumWikiTokensSeen() > numSeen){
    int currPos = scanner.yychar();
    //append whitespace
    for (int i = 0; i < (currPos - lastPos); i++){
      buffer.append(' ');
    }
    numAdded = scanner.setText(buffer);
    setupSavedToken(scanner.getPositionIncrement(), type);
    tmp.add(captureState());
    numSeen++;
    lastPos = currPos + numAdded;
  }
  //trim the buffer
  // TODO: this is inefficient
  String s = buffer.toString().trim();
  termAtt.setEmpty().append(s);
  offsetAtt.setOffset(correctOffset(theStart), correctOffset(theStart + s.length()));
  flagsAtt.setFlags(UNTOKENIZED_TOKEN_FLAG);
  //The way the loop is written, we will have proceeded to the next token.  We need to pushback the scanner to lastPos
  if (tmpTokType != WikipediaTokenizerImpl.YYEOF){
    scanner.yypushback(scanner.yylength());
  }
  tokens = tmp.iterator();
}
 

import org.apache.lucene.util.AttributeSource; //导入方法依赖的package包/类
@Override
protected void swap(int i, int j) {
  AttributeSource.State tmp = buffered[i];
  buffered[i] = buffered[j];
  buffered[j] = tmp;
  
  int tmp2 = startOff[i];
  startOff[i] = startOff[j];
  startOff[j] = tmp2;
  
  tmp2 = posInc[i];
  posInc[i] = posInc[j];
  posInc[j] = tmp2;
}
 

import org.apache.lucene.util.AttributeSource; //导入方法依赖的package包/类
@Override
public final boolean incrementToken() {
  // lazy init the iterator
  if (it == null) {
    it = cachedStates.iterator();
  }

  if (!it.hasNext()) {
    return false;
  }
  
  AttributeSource.State state = (State) it.next();
  restoreState(state.clone());
  return true;
}
 

import org.apache.lucene.util.AttributeSource; //导入方法依赖的package包/类
private void collapseAndSaveTokens(int tokenType, String type) throws IOException {
  //collapse
  StringBuilder buffer = new StringBuilder(32);
  int numAdded = scanner.setText(buffer);
  //TODO: how to know how much whitespace to add
  int theStart = scanner.yychar();
  int lastPos = theStart + numAdded;
  int tmpTokType;
  int numSeen = 0;
  List<AttributeSource.State> tmp = new ArrayList<AttributeSource.State>();
  setupSavedToken(0, type);
  tmp.add(captureState());
  //while we can get a token and that token is the same type and we have not transitioned to a new wiki-item of the same type
  while ((tmpTokType = scanner.getNextToken()) != WikipediaTokenizerImpl.YYEOF && tmpTokType == tokenType && scanner.getNumWikiTokensSeen() > numSeen){
    int currPos = scanner.yychar();
    //append whitespace
    for (int i = 0; i < (currPos - lastPos); i++){
      buffer.append(' ');
    }
    numAdded = scanner.setText(buffer);
    setupSavedToken(scanner.getPositionIncrement(), type);
    tmp.add(captureState());
    numSeen++;
    lastPos = currPos + numAdded;
  }
  //trim the buffer
  // TODO: this is inefficient
  String s = buffer.toString().trim();
  termAtt.setEmpty().append(s);
  offsetAtt.setOffset(correctOffset(theStart), correctOffset(theStart + s.length()));
  flagsAtt.setFlags(UNTOKENIZED_TOKEN_FLAG);
  //The way the loop is written, we will have proceeded to the next token.  We need to pushback the scanner to lastPos
  if (tmpTokType != WikipediaTokenizerImpl.YYEOF){
    scanner.yypushback(scanner.yylength());
  }
  tokens = tmp.iterator();
}
 

import org.apache.lucene.util.AttributeSource; //导入方法依赖的package包/类
@Override
public final boolean incrementToken() {
  // lazy init the iterator
  if (it == null) {
    it = cachedStates.iterator();
  }

  if (!it.hasNext()) {
    return false;
  }
  
  AttributeSource.State state = it.next();
  restoreState(state);
  return true;
}
 

import org.apache.lucene.util.AttributeSource; //导入方法依赖的package包/类
private void addState(AttributeSource.State state) {
  if (it != null) {
    throw new IllegalStateException("The tee must be consumed before sinks are consumed.");
  }
  cachedStates.add(state);
}
 

import org.apache.lucene.util.AttributeSource; //导入方法依赖的package包/类
private void setFinalState(AttributeSource.State finalState) {
  this.finalState = finalState;
}
 

import org.apache.lucene.util.AttributeSource; //导入方法依赖的package包/类
public void add(AttributeSource.State state) {
  inputTokens.add(state);
}
 

import org.apache.lucene.util.AttributeSource; //导入方法依赖的package包/类
public AttributeSource.State nextState() {
  assert nextRead < inputTokens.size();
  return inputTokens.get(nextRead++);
}
 

import org.apache.lucene.util.AttributeSource; //导入方法依赖的package包/类
private static AttributeSource.State createState(AttributeSource a, Tok state, int tokenEnd) {
  a.clearAttributes();
  CharTermAttribute termAtt = a.addAttribute(CharTermAttribute.class);
  char[] tokChars = state.token.toString().toCharArray();
  termAtt.copyBuffer(tokChars, 0, tokChars.length);
  int tokenStart = tokenEnd - state.token.length();
  for (Entry<String, String> e : state.attr.entrySet()) {
    String k = e.getKey();
    if (k.equals("i")) {
      // position increment
      int incr = Integer.parseInt(e.getValue());
      PositionIncrementAttribute posIncr = a.addAttribute(PositionIncrementAttribute.class);
      posIncr.setPositionIncrement(incr);
    } else if (k.equals("s")) {
      tokenStart = Integer.parseInt(e.getValue());
    } else if (k.equals("e")) {
      tokenEnd = Integer.parseInt(e.getValue());
    } else if (k.equals("y")) {
      TypeAttribute type = a.addAttribute(TypeAttribute.class);
      type.setType(e.getValue());
    } else if (k.equals("f")) {
      FlagsAttribute flags = a.addAttribute(FlagsAttribute.class);
      int f = Integer.parseInt(e.getValue(), 16);
      flags.setFlags(f);
    } else if (k.equals("p")) {
      PayloadAttribute p = a.addAttribute(PayloadAttribute.class);
      byte[] data = hexToBytes(e.getValue());
      if (data != null && data.length > 0) {
        p.setPayload(new BytesRef(data));
      }
    } else {
      // unknown attribute
    }
  }
  // handle offset attr
  OffsetAttribute offset = a.addAttribute(OffsetAttribute.class);
  offset.setOffset(tokenStart, tokenEnd);
  State resState = a.captureState();
  a.clearAttributes();
  return resState;
}
 

import org.apache.lucene.util.AttributeSource; //导入方法依赖的package包/类
public AttributeSource.State nextState() {
	assert nextRead < inputTokens.size();
	return inputTokens.get(nextRead++);
}