Java AttributeSource类代码示例

import org.apache.lucene.util.AttributeSource; //导入依赖的package包/类
@Override
public boolean incrementToken() throws IOException {
  if (input.incrementToken()) {
    // capture state lazily - maybe no SinkFilter accepts this state
    AttributeSource.State state = null;
    for (WeakReference<SinkTokenStream> ref : sinks) {
      final SinkTokenStream sink = ref.get();
      if (sink != null) {
        if (sink.accept(this)) {
          if (state == null) {
            state = this.captureState();
          }
          sink.addState(state);
        }
      }
    }
    return true;
  }
  
  return false;
}
 

import org.apache.lucene.util.AttributeSource; //导入依赖的package包/类
@Override
public boolean accept(AttributeSource source) {
  if (termAtt == null) {
    termAtt = source.addAttribute(CharTermAttribute.class);
  }
  try {
    Date date = dateFormat.parse(termAtt.toString());//We don't care about the date, just that we can parse it as a date
    if (date != null) {
      return true;
    }
  } catch (ParseException e) {

  }
  
  return false;
}
 

import org.apache.lucene.util.AttributeSource; //导入依赖的package包/类
@Override
public final boolean incrementToken() throws IOException {
  if (cache == null) {
    // fill cache lazily
    cache = new LinkedList<AttributeSource.State>();
    fillCache();
    iterator = cache.iterator();
  }
  
  if (!iterator.hasNext()) {
    // the cache is exhausted, return false
    return false;
  }
  // Since the TokenFilter can be reset, the tokens need to be preserved as immutable.
  restoreState(iterator.next());
  return true;
}
 

import org.apache.lucene.util.AttributeSource; //导入依赖的package包/类
private String[] walkTokens() throws IOException {
    List<String> wordList = new ArrayList<>();
    while (input.incrementToken()) {
        CharTermAttribute textAtt = input.getAttribute(CharTermAttribute.class);
        OffsetAttribute offsetAtt = input.getAttribute(OffsetAttribute.class);
        char[] buffer = textAtt.buffer();
        String word = new String(buffer, 0, offsetAtt.endOffset() - offsetAtt.startOffset());
        wordList.add(word);
        AttributeSource attrs = input.cloneAttributes();
        tokenAttrs.add(attrs);
    }
    String[] words = new String[wordList.size()];
    for (int i = 0; i < words.length; i++) {
        words[i] = wordList.get(i);
    }
    return words;
}
 

import org.apache.lucene.util.AttributeSource; //导入依赖的package包/类
TokenizerWrapper() {
    super();

    tokenizerTimestamp = dictionaryTimestamp;
    tokenizer = new JapaneseTokenizer(userDictionary,
            discartPunctuation, mode);

    try {
        final Field attributesField = getAccessibleField(AttributeSource.class, "attributes");
        final Object attributesObj = attributesField.get(tokenizer);
        attributesField.set(this, attributesObj);

        final Field attributeImplsField = getAccessibleField(AttributeSource.class, "attributeImpls");
        final Object attributeImplsObj = attributeImplsField.get(tokenizer);
        attributeImplsField.set(this, attributeImplsObj);

        final Field currentStateField = getAccessibleField(AttributeSource.class, "currentState");
        final Object currentStateObj = currentStateField.get(tokenizer);
        currentStateField.set(this, currentStateObj);
    } catch (final Exception e) {
        throw new IllegalStateException(
                "Failed to update the tokenizer.", e);
    }
}
 

import org.apache.lucene.util.AttributeSource; //导入依赖的package包/类
static Object[] newTokenizerArgs(Random random, Reader reader, Class<?>[] paramTypes) {
  Object[] args = new Object[paramTypes.length];
  for (int i = 0; i < args.length; i++) {
    Class<?> paramType = paramTypes[i];
    if (paramType == Reader.class) {
      args[i] = reader;
    } else if (paramType == AttributeSource.class) {
      // TODO: args[i] = new AttributeSource();
      // this is currently too scary to deal with!
      args[i] = null; // force IAE
    } else {
      args[i] = newRandomArg(random, paramType);
    }
  }
  return args;
}
 

import org.apache.lucene.util.AttributeSource; //导入依赖的package包/类
/**
 * Analyzes the given TokenStream, collecting the Tokens it produces.
 *
 * @param tokenStream TokenStream to analyze
 *
 * @return List of tokens produced from the TokenStream
 */
private List<AttributeSource> analyzeTokenStream(TokenStream tokenStream) {
  final List<AttributeSource> tokens = new ArrayList<>();
  final PositionIncrementAttribute posIncrAtt = tokenStream.addAttribute(PositionIncrementAttribute.class);
  final TokenTrackingAttribute trackerAtt = tokenStream.addAttribute(TokenTrackingAttribute.class);
  // for backwards compatibility, add all "common" attributes
  tokenStream.addAttribute(OffsetAttribute.class);
  tokenStream.addAttribute(TypeAttribute.class);
  try {
    tokenStream.reset();
    int position = 0;
    while (tokenStream.incrementToken()) {
      position += posIncrAtt.getPositionIncrement();
      trackerAtt.setActPosition(position);
      tokens.add(tokenStream.cloneAttributes());
    }
    tokenStream.end();
  } catch (IOException ioe) {
    throw new RuntimeException("Error occured while iterating over tokenstream", ioe);
  } finally {
    IOUtils.closeWhileHandlingException(tokenStream);
  }

  return tokens;
}
 

import org.apache.lucene.util.AttributeSource; //导入依赖的package包/类
TokenizerWrapper() {
    super();

    tokenizerTimestamp = dictionaryTimestamp;
    tokenizer = new JapaneseTokenizer(userDictionary, discartPunctuation, mode);

    try {
        Field attributesField = getAccessibleField(AttributeSource.class, "attributes");
        final Object attributesObj = attributesField.get(tokenizer);
        attributesField.set(this, attributesObj);

        Field attributeImplsField = getAccessibleField(AttributeSource.class, "attributeImpls");
        final Object attributeImplsObj = attributeImplsField.get(tokenizer);
        attributeImplsField.set(this, attributeImplsObj);

        Field currentStateField = getAccessibleField(AttributeSource.class, "currentState");
        final Object currentStateObj = currentStateField.get(tokenizer);
        currentStateField.set(this, currentStateObj);
    } catch (final Exception e) {
        throw new IllegalStateException(
                "Failed to update the tokenizer.", e);
    }
}
 

import org.apache.lucene.util.AttributeSource; //导入依赖的package包/类
static Object[] newTokenizerArgs(Random random, Reader reader, Class<?>[] paramTypes) {
  Object[] args = new Object[paramTypes.length];
  for (int i = 0; i < args.length; i++) {
    Class<?> paramType = paramTypes[i];
    if (paramType == Reader.class) {
      args[i] = reader;
    } else if (paramType == AttributeFactory.class) {
      // TODO: maybe the collator one...???
      args[i] = AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY;
    } else if (paramType == AttributeSource.class) {
      // TODO: args[i] = new AttributeSource();
      // this is currently too scary to deal with!
      args[i] = null; // force IAE
    } else {
      args[i] = newRandomArg(random, paramType);
    }
  }
  return args;
}
 

import org.apache.lucene.util.AttributeSource; //导入依赖的package包/类
@Override @SuppressWarnings("unchecked")
protected TermsEnum getTermsEnum(final Terms terms, AttributeSource atts) throws IOException {
  // very strange: java.lang.Number itself is not Comparable, but all subclasses used here are
  if (min != null && max != null && ((Comparable<T>) min).compareTo(max) > 0) {
    return TermsEnum.EMPTY;
  }
  return new NumericRangeTermsEnum(terms.iterator(null));
}
 

import org.apache.lucene.util.AttributeSource; //导入依赖的package包/类
@Override
protected TermsEnum getTermsEnum(Terms terms, AttributeSource atts) throws IOException {
  if (lowerTerm != null && upperTerm != null && lowerTerm.compareTo(upperTerm) > 0) {
    return TermsEnum.EMPTY;
  }
  
  TermsEnum tenum = terms.iterator(null);
  
  if ((lowerTerm == null || (includeLower && lowerTerm.length == 0)) && upperTerm == null) {
    return tenum;
  }
  return new TermRangeTermsEnum(tenum,
      lowerTerm, upperTerm, includeLower, includeUpper);
}
 

import org.apache.lucene.util.AttributeSource; //导入依赖的package包/类
@Override  
protected TermsEnum getTermsEnum(Terms terms, AttributeSource atts) throws IOException {
  TermsEnum tenum = terms.iterator(null);
  
  if (prefix.bytes().length == 0) {
    // no prefix -- match all terms for this field:
    return tenum;
  }
  return new PrefixTermsEnum(tenum, prefix.bytes());
}
 

import org.apache.lucene.util.AttributeSource; //导入依赖的package包/类
@Override
protected TermsEnum getTermsEnum(Terms terms, AttributeSource atts) throws IOException {
  if (maxEdits == 0 || prefixLength >= term.text().length()) {  // can only match if it's exact
    return new SingleTermsEnum(terms.iterator(null), term.bytes());
  }
  return new FuzzyTermsEnum(terms, atts, getTerm(), maxEdits, prefixLength, transpositions);
}
 

import org.apache.lucene.util.AttributeSource; //导入依赖的package包/类
/**
 * Sets attributeSource to a new instance.
 */
void setAttributeSource(AttributeSource attributeSource) {
  if (this.attributeSource != attributeSource) {
    this.attributeSource = attributeSource;
    termAttribute = attributeSource.getAttribute(TermToBytesRefAttribute.class);
    posIncrAttribute = attributeSource.addAttribute(PositionIncrementAttribute.class);
    offsetAttribute = attributeSource.addAttribute(OffsetAttribute.class);
    payloadAttribute = attributeSource.getAttribute(PayloadAttribute.class);
  }
}
 

import org.apache.lucene.util.AttributeSource; //导入依赖的package包/类
private AttributeSource nextTok() throws IOException {
  if (buffer!=null && !buffer.isEmpty()) {
    return buffer.removeFirst();
  } else {
    if (!exhausted && input.incrementToken()) {
      return this;
    } else {
      exhausted = true;
      return null;
    }
  }
}