Java TokenFilterFactory類代碼示例

import org.apache.lucene.analysis.util.TokenFilterFactory; //導入依賴的package包/類
public void testIgnoreWhitespace() throws Exception {
  String withSpace = "foo bar";
  String withoutSpace = "foobar";
  String withPunctuation = "foo-bar";
  TokenFilterFactory factory = tokenFilterFactory("ICUCollationKey",
      "locale", "en",
      "strength", "primary",
      "alternate", "shifted",
      "variableTop", " ");
  TokenStream tsWithSpace = factory.create(
      new KeywordTokenizer(new StringReader(withSpace)));
  TokenStream tsWithoutSpace = factory.create(
      new KeywordTokenizer(new StringReader(withoutSpace)));
  assertCollatesToSame(tsWithSpace, tsWithoutSpace);
  // now assert that punctuation still matters: foo-bar < foo bar
  tsWithSpace = factory.create(
      new KeywordTokenizer(new StringReader(withSpace)));
  TokenStream tsWithPunctuation = factory.create(
      new KeywordTokenizer(new StringReader(withPunctuation)));
  assertCollation(tsWithPunctuation, tsWithSpace, -1);
}
 

import org.apache.lucene.analysis.util.TokenFilterFactory; //導入依賴的package包/類
private void doTestTokenFilter(String tokenfilter) throws IOException {
  Class<? extends TokenFilterFactory> factoryClazz = TokenFilterFactory.lookupClass(tokenfilter);
  TokenFilterFactory factory = (TokenFilterFactory) initialize(factoryClazz);
  if (factory != null) {
    // we managed to fully create an instance. check a few more things:
    
    // if it implements MultiTermAware, sanity check its impl
    if (factory instanceof MultiTermAwareComponent) {
      AbstractAnalysisFactory mtc = ((MultiTermAwareComponent) factory).getMultiTermComponent();
      assertNotNull(mtc);
      // its not ok to return a charfilter or tokenizer here, this makes no sense
      assertTrue(mtc instanceof TokenFilterFactory);
    }
    
    // beast it just a little, it shouldnt throw exceptions:
    // (it should have thrown them in initialize)
    checkRandomData(random(), new FactoryAnalyzer(assertingTokenizer, factory, null), 100, 20, false, false);
  }
}
 

import org.apache.lucene.analysis.util.TokenFilterFactory; //導入依賴的package包/類
@Before
public void setUp() {
   fieldType = new TextField();

   Map<String, FieldType> fieldTypes = Maps.newHashMap();
   fieldTypes.put("test", fieldType);

   when(searcher.getSchema()).thenReturn(schema);
   when(schema.getFieldTypes()).thenReturn(fieldTypes);

   indexAnalyzer = new TokenizerChain(
         new WhitespaceTokenizerFactory(Maps.<String, String> newHashMap()),
         new TokenFilterFactory[] { indexTokenFilterFactory });
   queryAnalyzer = new TokenizerChain(
         new WhitespaceTokenizerFactory(Maps.<String, String> newHashMap()),
         new TokenFilterFactory[] { queryTokenFilterFactory });

   reloader = new SearcherAwareReloader(null);
}
 

import org.apache.lucene.analysis.util.TokenFilterFactory; //導入依賴的package包/類
private void doTestTokenFilter(String tokenfilter) throws IOException {
  TokenFilterFactory factory = TokenFilterFactory.forName(tokenfilter);
  if (initialize(factory)) {
    // we managed to fully create an instance. check a few more things:
    
    // if it implements MultiTermAware, sanity check its impl
    if (factory instanceof MultiTermAwareComponent) {
      AbstractAnalysisFactory mtc = ((MultiTermAwareComponent) factory).getMultiTermComponent();
      assertNotNull(mtc);
      // its not ok to return a charfilter or tokenizer here, this makes no sense
      assertTrue(mtc instanceof TokenFilterFactory);
    }
    
    // beast it just a little, it shouldnt throw exceptions:
    // (it should have thrown them in initialize)
    checkRandomData(random(), new FactoryAnalyzer(assertingTokenizer, factory, null), 100, 20, false, false);
  }
}
 

import org.apache.lucene.analysis.util.TokenFilterFactory; //導入依賴的package包/類
/**
 * Reloads all Lucene SPI implementations using the new classloader.
 * This method must be called after the new classloader has been created to
 * register the services for use.
 */
static void reloadLuceneSPI(ClassLoader loader) {
    // do NOT change the order of these method calls!

    // Codecs:
    PostingsFormat.reloadPostingsFormats(loader);
    DocValuesFormat.reloadDocValuesFormats(loader);
    Codec.reloadCodecs(loader);
    // Analysis:
    CharFilterFactory.reloadCharFilters(loader);
    TokenFilterFactory.reloadTokenFilters(loader);
    TokenizerFactory.reloadTokenizers(loader);
}
 

import org.apache.lucene.analysis.util.TokenFilterFactory; //導入依賴的package包/類
/**
 * This method looks up a class with its fully qualified name (FQN), or a short-name
 * class-simplename, or with a package suffix, assuming "org.apache.lucene.analysis."
 * as the package prefix (e.g. "standard.ClassicTokenizerFactory" ->
 * "org.apache.lucene.analysis.standard.ClassicTokenizerFactory").
 *
 * If className contains a period, the class is first looked up as-is, assuming that it
 * is an FQN.  If this fails, lookup is retried after prepending the Lucene analysis
 * package prefix to the class name.
 *
 * If className does not contain a period, the analysis SPI *Factory.lookupClass()
 * methods are used to find the class.
 *
 * @param className The name or the short name of the class.
 * @param expectedType The superclass className is expected to extend
 * @return the loaded class.
 * @throws ClassNotFoundException if lookup fails
 */
public <T> Class<? extends T> lookupAnalysisClass(String className, Class<T> expectedType)
    throws ClassNotFoundException {
  if (className.contains(".")) {
    try {
      // First, try className == FQN
      return Class.forName(className).asSubclass(expectedType);
    } catch (ClassNotFoundException e) {
      try {
        // Second, retry lookup after prepending the Lucene analysis package prefix
        return Class.forName(LUCENE_ANALYSIS_PACKAGE_PREFIX + className).asSubclass(expectedType);
      } catch (ClassNotFoundException e1) {
        throw new ClassNotFoundException("Can't find class '" + className
                                         + "' or '" + LUCENE_ANALYSIS_PACKAGE_PREFIX + className + "'");
      }
    }
  }
  // No dot - use analysis SPI lookup
  final String analysisComponentName = ANALYSIS_COMPONENT_SUFFIX_PATTERN.matcher(className).replaceFirst("");
  if (CharFilterFactory.class.isAssignableFrom(expectedType)) {
    return CharFilterFactory.lookupClass(analysisComponentName).asSubclass(expectedType);
  } else if (TokenizerFactory.class.isAssignableFrom(expectedType)) {
    return TokenizerFactory.lookupClass(analysisComponentName).asSubclass(expectedType);
  } else if (TokenFilterFactory.class.isAssignableFrom(expectedType)) {
    return TokenFilterFactory.lookupClass(analysisComponentName).asSubclass(expectedType);
  }

  throw new ClassNotFoundException("Can't find class '" + className + "'");
}
 

import org.apache.lucene.analysis.util.TokenFilterFactory; //導入依賴的package包/類
public AnalyzerFactory(List<CharFilterFactory> charFilterFactories,
                       TokenizerFactory tokenizerFactory,
                       List<TokenFilterFactory> tokenFilterFactories) {
  this.charFilterFactories = charFilterFactories;
  assert null != tokenizerFactory;
  this.tokenizerFactory = tokenizerFactory;
  this.tokenFilterFactories = tokenFilterFactories;
}
 

import org.apache.lucene.analysis.util.TokenFilterFactory; //導入依賴的package包/類
@Override
public String toString() {
  StringBuilder sb = new StringBuilder("AnalyzerFactory(");
  if (null != name) {
    sb.append("name:");
    sb.append(name);
    sb.append(", ");
  }
  if (null != positionIncrementGap) {
    sb.append("positionIncrementGap:");
    sb.append(positionIncrementGap);
    sb.append(", ");
  }
  if (null != offsetGap) {
    sb.append("offsetGap:");
    sb.append(offsetGap);
    sb.append(", ");
  }
  for (CharFilterFactory charFilterFactory: charFilterFactories) {
    sb.append(charFilterFactory);
    sb.append(", ");
  }
  sb.append(tokenizerFactory);
  for (TokenFilterFactory tokenFilterFactory : tokenFilterFactories) {
    sb.append(", ");
    sb.append(tokenFilterFactory);
  }
  sb.append(')');
  return sb.toString();
}
 

import org.apache.lucene.analysis.util.TokenFilterFactory; //導入依賴的package包/類
public void testBasicUsage() throws Exception {
  String turkishUpperCase = "I WİLL USE TURKİSH CASING";
  String turkishLowerCase = "ı will use turkish casıng";
  TokenFilterFactory factory = tokenFilterFactory("ICUCollationKey",
      "locale", "tr",
      "strength", "primary");
  TokenStream tsUpper = factory.create(
      new KeywordTokenizer(new StringReader(turkishUpperCase)));
  TokenStream tsLower = factory.create(
      new KeywordTokenizer(new StringReader(turkishLowerCase)));
  assertCollatesToSame(tsUpper, tsLower);
}
 

import org.apache.lucene.analysis.util.TokenFilterFactory; //導入依賴的package包/類
public void testNormalization() throws Exception {
  String turkishUpperCase = "I W\u0049\u0307LL USE TURKİSH CASING";
  String turkishLowerCase = "ı will use turkish casıng";
  TokenFilterFactory factory = tokenFilterFactory("ICUCollationKey",
      "locale", "tr",
      "strength", "primary",
      "decomposition", "canonical");
  TokenStream tsUpper = factory.create(
      new KeywordTokenizer(new StringReader(turkishUpperCase)));
  TokenStream tsLower = factory.create(
      new KeywordTokenizer(new StringReader(turkishLowerCase)));
  assertCollatesToSame(tsUpper, tsLower);
}
 

import org.apache.lucene.analysis.util.TokenFilterFactory; //導入依賴的package包/類
public void testSecondaryStrength() throws Exception {
  String upperCase = "TESTING";
  String lowerCase = "testing";
  TokenFilterFactory factory = tokenFilterFactory("ICUCollationKey",
      "locale", "en",
      "strength", "secondary",
      "decomposition", "no");
  TokenStream tsUpper = factory.create(
      new KeywordTokenizer(new StringReader(upperCase)));
  TokenStream tsLower = factory.create(
      new KeywordTokenizer(new StringReader(lowerCase)));
  assertCollatesToSame(tsUpper, tsLower);
}
 

import org.apache.lucene.analysis.util.TokenFilterFactory; //導入依賴的package包/類
public void testIgnorePunctuation() throws Exception {
  String withPunctuation = "foo-bar";
  String withoutPunctuation = "foo bar";
  TokenFilterFactory factory = tokenFilterFactory("ICUCollationKey",
      "locale", "en",
      "strength", "primary",
      "alternate", "shifted");
  TokenStream tsPunctuation = factory.create(
      new KeywordTokenizer(new StringReader(withPunctuation)));
  TokenStream tsWithoutPunctuation = factory.create(
      new KeywordTokenizer(new StringReader(withoutPunctuation)));
  assertCollatesToSame(tsPunctuation, tsWithoutPunctuation);
}
 

import org.apache.lucene.analysis.util.TokenFilterFactory; //導入依賴的package包/類
public void testNumerics() throws Exception {
  String nine = "foobar-9";
  String ten = "foobar-10";
  TokenFilterFactory factory = tokenFilterFactory("ICUCollationKey",
      "locale", "en",
      "numeric", "true");
  TokenStream tsNine = factory.create(
      new KeywordTokenizer(new StringReader(nine)));
  TokenStream tsTen = factory.create(
      new KeywordTokenizer(new StringReader(ten)));
  assertCollation(tsNine, tsTen, -1);
}
 

import org.apache.lucene.analysis.util.TokenFilterFactory; //導入依賴的package包/類
public void testIgnoreAccentsButNotCase() throws Exception {
  String withAccents = "résumé";
  String withoutAccents = "resume";
  String withAccentsUpperCase = "Résumé";
  String withoutAccentsUpperCase = "Resume";
  TokenFilterFactory factory = tokenFilterFactory("ICUCollationKey",
      "locale", "en",
      "strength", "primary",
      "caseLevel", "true");
  TokenStream tsWithAccents = factory.create(
      new KeywordTokenizer(new StringReader(withAccents)));
  TokenStream tsWithoutAccents = factory.create(
      new KeywordTokenizer(new StringReader(withoutAccents)));
  assertCollatesToSame(tsWithAccents, tsWithoutAccents);
  
  TokenStream tsWithAccentsUpperCase = factory.create(
      new KeywordTokenizer(new StringReader(withAccentsUpperCase)));
  TokenStream tsWithoutAccentsUpperCase = factory.create(
      new KeywordTokenizer(new StringReader(withoutAccentsUpperCase)));
  assertCollatesToSame(tsWithAccentsUpperCase, tsWithoutAccentsUpperCase);
  
  // now assert that case still matters: resume < Resume
  TokenStream tsLower = factory.create(
      new KeywordTokenizer(new StringReader(withoutAccents)));
  TokenStream tsUpper = factory.create(
      new KeywordTokenizer(new StringReader(withoutAccentsUpperCase)));
  assertCollation(tsLower, tsUpper, -1);
}
 

import org.apache.lucene.analysis.util.TokenFilterFactory; //導入依賴的package包/類
public void testUpperCaseFirst() throws Exception {
  String lower = "resume";
  String upper = "Resume";
  TokenFilterFactory factory = tokenFilterFactory("ICUCollationKey",
      "locale", "en",
      "strength", "tertiary",
      "caseFirst", "upper");
  TokenStream tsLower = factory.create(
      new KeywordTokenizer(new StringReader(lower)));
  TokenStream tsUpper = factory.create(
      new KeywordTokenizer(new StringReader(upper)));
  assertCollation(tsUpper, tsLower, -1);
}