Java WhitespaceTokenizer.INSTANCE属性代码示例

public static DoccatModel getNLPModel(File openNLPTraining) throws IOException {
	DoccatModel model = null;

	FeatureGenerator[] def = { new BagOfWordsFeatureGenerator() };
	WhitespaceTokenizer tokenizer = WhitespaceTokenizer.INSTANCE;

	DoccatFactory factory = new DoccatFactory(tokenizer, def);
	InputStreamFactory isf = new MarkableFileInputStreamFactory(openNLPTraining);
	ObjectStream<String> lineStream = new PlainTextByLineStream(isf, "UTF-8");
	ObjectStream<DocumentSample> sampleStream = new DocumentSampleStream(lineStream);

	TrainingParameters params = TrainingParameters.defaultParams();
	System.out.println(params.algorithm());
	params.put(TrainingParameters.CUTOFF_PARAM, Integer.toString(0));
	params.put(TrainingParameters.ITERATIONS_PARAM, Integer.toString(4000));

	model = DocumentCategorizerME.train("en", sampleStream, params, factory);
	
	evaluateDoccatModel(model, openNLPTraining);

	return model;

}
 

@Override
 public IntentMatcher getIntentMatcher()
 {
// model was built with OpenNLP whitespace tokenizer
   OpenNLPTokenizer tokenizer = new OpenNLPTokenizer(WhitespaceTokenizer.INSTANCE);
   
   // use Open NLP NER for slot matching
   OpenNLPSlotMatcher slotMatcher = new OpenNLPSlotMatcher(tokenizer);
   slotMatcher.addSlotModel("Address", "models/en-ner-address.bin");   

   // create intent matcher
   OpenNLPIntentMatcher matcher = new OpenNLPIntentMatcher("models/en-cat-taxi-intents.bin", tokenizer, slotMatcher);

   Intent intent = new Intent("OrderTaxi");
   intent.addSlot(new LiteralSlot("Address"));
   matcher.addIntent(intent);

   intent = new Intent("CancelTaxi");
   matcher.addIntent(intent);

   intent = new Intent("WhereTaxi");
   matcher.addIntent(intent);

   intent = new Intent("GaveAddress");
   intent.addSlot(new LiteralSlot("Address"));
   matcher.addIntent(intent);

   intent = new Intent("Stop");
   matcher.addIntent(intent);

   intent = new Intent("Help");
   matcher.addIntent(intent);

   intent = new Intent("FavColor");
   matcher.addIntent(intent);

   return matcher;
 }
 

@Override
 public IntentMatcher getIntentMatcher()
 {
// model was built with OpenNLP whitespace tokenizer
   OpenNLPTokenizer tokenizer = new OpenNLPTokenizer(WhitespaceTokenizer.INSTANCE);
   
   // use Open NLP NER for slot matching
   OpenNLPSlotMatcher slotMatcher = new OpenNLPSlotMatcher(tokenizer);
   slotMatcher.addSlotModel("Address", "models/en-ner-address.bin");    

   // create intent matcher
   OpenNLPIntentMatcher matcher = new OpenNLPIntentMatcher("models/en-cat-taxi-intents.bin", tokenizer, slotMatcher);

   Intent intent = new Intent("OrderTaxi");
   intent.addSlot(new LiteralSlot("Address"));
   matcher.addIntent(intent);

   intent = new Intent("CancelTaxi");
   matcher.addIntent(intent);

   intent = new Intent("WhereTaxi");
   matcher.addIntent(intent);

   intent = new Intent("GaveAddress");
   intent.addSlot(new LiteralSlot("Address"));
   matcher.addIntent(intent);

   return matcher;
 }
 

public Tokenizer getTokenizer() {
if (this.tokenizer == null) {
    if (artifactProvider != null) {
	String className = artifactProvider.getManifestProperty(TOKENIZER_NAME);
	if (className != null) {
	    this.tokenizer = ExtensionLoader.instantiateExtension(
	        Tokenizer.class , className);
	}
    }
    if (this.tokenizer == null) { // could not load using artifact provider
	this.tokenizer = WhitespaceTokenizer.INSTANCE;
    }
}
return tokenizer;
   }
 

@Test
public void testCategorization()
  throws Exception
{
  URL modelUrl = Thread.currentThread().getContextClassLoader()
      .getResource("models/en-cat-taxi-intents.bin");
  assertThat(modelUrl, is(notNullValue()));

  DoccatModel model = new DoccatModel(modelUrl);
  assertThat(model, is(notNullValue()));

  DocumentCategorizerME myCategorizer = new DocumentCategorizerME(model);
  // model was built with OpenNLP whitespace tokenizer
  OpenNLPTokenizer tokenizer = new OpenNLPTokenizer(WhitespaceTokenizer.INSTANCE);

  String category = myCategorizer.getBestCategory(myCategorizer.categorize(tokenizer.tokenize("Order me a taxi")));
  assertThat(category, is(notNullValue()));
  assertThat(category, is("OrderTaxi"));

  category = myCategorizer.getBestCategory(myCategorizer.categorize(tokenizer.tokenize("Send me a taxi")));
  assertThat(category, is(notNullValue()));
  assertThat(category, is("OrderTaxi"));

  category = myCategorizer
      .getBestCategory(myCategorizer.categorize(tokenizer.tokenize("Send a taxi to 12 Pleasent Street")));
  assertThat(category, is(notNullValue()));
  assertThat(category, is("OrderTaxi"));

  category = myCategorizer.getBestCategory(myCategorizer.categorize(tokenizer.tokenize("Cancel my cab")));
  assertThat(category, is(notNullValue()));
  assertThat(category, is("CancelTaxi"));

  category = myCategorizer.getBestCategory(myCategorizer.categorize(tokenizer.tokenize("Where is my taxi ?")));
  assertThat(category, is(notNullValue()));
  assertThat(category, is("WhereTaxi"));

  category = myCategorizer
      .getBestCategory(myCategorizer.categorize(tokenizer.tokenize("The address is 136 River Road")));
  assertThat(category, is(notNullValue()));
  assertThat(category, is("GaveAddress"));
}
 

public Tokenizer getTokenizer() {
if(tokenizer != null) {
    return ExtensionLoader.instantiateExtension(Tokenizer.class, this.tokenizer);
}
return WhitespaceTokenizer.INSTANCE;
   }
 

private static Tokenizer createTokenizer(String tokenizer) {
if(tokenizer != null) {
    return ExtensionLoader.instantiateExtension(Tokenizer.class, tokenizer);
}
return WhitespaceTokenizer.INSTANCE;
   }
 

private Tokenizer getTokenizer(String tokenizer) {
    if(tokenizer != null) {
        return ExtensionLoader.instantiateExtension(Tokenizer.class, tokenizer);
    }
    return WhitespaceTokenizer.INSTANCE;
}
 

public AuthorAgeSampleStream(ObjectStream<String> samples) {
super(samples);
this.tokenizer = WhitespaceTokenizer.INSTANCE;
   }