Java LibSvmStringOutcomeDataWriter类代码示例

import org.cleartk.ml.libsvm.LibSvmStringOutcomeDataWriter; //导入依赖的package包/类
@Override
public void train(CollectionReader collectionReader, File outputDirectory) throws Exception {
  AggregateBuilder builder = new AggregateBuilder();
  builder.add(UriToDocumentTextAnnotator.getDescription());
  builder.add(SentenceAnnotator.getDescription());
  builder.add(TokenAnnotator.getDescription());
  builder.add(PosTaggerAnnotator.getDescription());
  builder.add(DefaultSnowballStemmer.getDescription("English"));
  builder.add(AnalysisEngineFactory.createEngineDescription(GoldQuestionCategoryAnnotator.class));
  AnalysisEngineDescription documentClassificationAnnotator = AnalysisEngineFactory.createEngineDescription(
      QuestionCategoryAnnotator.class, CleartkAnnotator.PARAM_IS_TRAINING, true,
      DirectoryDataWriterFactory.PARAM_OUTPUT_DIRECTORY, outputDirectory,
      DefaultDataWriterFactory.PARAM_DATA_WRITER_CLASS_NAME, LibSvmStringOutcomeDataWriter.class.getName());
  builder.add(documentClassificationAnnotator);
  SimplePipeline.runPipeline(collectionReader, builder.createAggregateDescription());
  System.err.println("Train model and write model.jar file.");
  HideOutput hider = new HideOutput();
  Train.main(outputDirectory, this.trainingArguments.toArray(new String[this.trainingArguments.size()]));
  hider.restoreOutput();
}
 

import org.cleartk.ml.libsvm.LibSvmStringOutcomeDataWriter; //导入依赖的package包/类
@Override
protected void train(CollectionReader collectionReader, File directory) throws Exception {
	String tmpView = "DP";

	AggregateBuilder b = new AggregateBuilder();

	b.add(AnalysisEngineFactory.createEngineDescription(PrepareClearTk.class, PrepareClearTk.PARAM_VIEW_NAME,
			tmpView, PrepareClearTk.PARAM_ANNOTATION_TYPE, DramatisPersonae.class,
			PrepareClearTk.PARAM_SUBANNOTATIONS, Arrays.asList(Figure.class)));
	b.add(AnalysisEngineFactory.createEngineDescription(BreakIteratorSegmenter.class), CAS.NAME_DEFAULT_SOFA,
			tmpView);
	b.add(AnalysisEngineFactory.createEngineDescription(ClearTkGenderAnnotator.class,
			DefaultDataWriterFactory.PARAM_DATA_WRITER_CLASS_NAME, LibSvmStringOutcomeDataWriter.class,
			DirectoryDataWriterFactory.PARAM_OUTPUT_DIRECTORY, directory), CAS.NAME_DEFAULT_SOFA, tmpView);
	b.add(AnalysisEngineFactory.createEngineDescription(XmiWriter.class, XmiWriter.PARAM_TARGET_LOCATION,
			"target/xmi"));

	SimplePipeline.runPipeline(collectionReader, b.createAggregate());

	Train.main(directory, new String[] { "-t", "0" });
}
 

import org.cleartk.ml.libsvm.LibSvmStringOutcomeDataWriter; //导入依赖的package包/类
public static void main(String[] args) throws Exception {
	CollectionReaderDescription reader = CollectionReaderFactory.createReaderDescription(XmiReader.class,
			XmiReader.PARAM_SOURCE_LOCATION, "src/main/resources/gender/*/*.xmi", XmiReader.PARAM_LENIENT, true);

	String tmpView = "DP";

	AggregateBuilder b = new AggregateBuilder();

	b.add(AnalysisEngineFactory.createEngineDescription(PrepareClearTk.class, PrepareClearTk.PARAM_VIEW_NAME,
			tmpView, PrepareClearTk.PARAM_ANNOTATION_TYPE, DramatisPersonae.class,
			PrepareClearTk.PARAM_SUBANNOTATIONS, Arrays.asList(Figure.class)));
	b.add(AnalysisEngineFactory.createEngineDescription(BreakIteratorSegmenter.class), CAS.NAME_DEFAULT_SOFA,
			tmpView);
	b.add(AnalysisEngineFactory.createEngineDescription(ClearTkGenderAnnotator.class,
			DefaultDataWriterFactory.PARAM_DATA_WRITER_CLASS_NAME, LibSvmStringOutcomeDataWriter.class,
			DirectoryDataWriterFactory.PARAM_OUTPUT_DIRECTORY, "target/models"), CAS.NAME_DEFAULT_SOFA, tmpView);
	b.add(AnalysisEngineFactory.createEngineDescription(XmiWriter.class, XmiWriter.PARAM_TARGET_LOCATION,
			"target/xmi"));
	SimplePipeline.runPipeline(reader, b.createAggregateDescription());

	Train.main(new File("target/models"), new String[] { "-t", "0" });
}
 

import org.cleartk.ml.libsvm.LibSvmStringOutcomeDataWriter; //导入依赖的package包/类
@Test
public void testLibsvm() throws Exception {
  String outDirectoryName = outputDirectoryName + "/libsvm";

  AnalysisEngineDescription dataWriter = AnalysisEngineFactory.createEngineDescription(
      ExamplePosAnnotator.class,
      CleartkSequenceAnnotator.PARAM_DATA_WRITER_FACTORY_CLASS_NAME,
      ViterbiDataWriterFactory.class.getName(),
      DirectoryDataWriterFactory.PARAM_OUTPUT_DIRECTORY,
      outDirectoryName,
      DefaultDataWriterFactory.PARAM_DATA_WRITER_CLASS_NAME,
      LibSvmStringOutcomeDataWriter.class.getName(),
      ViterbiDataWriterFactory.PARAM_OUTCOME_FEATURE_EXTRACTOR_NAMES,
      new String[] { DefaultOutcomeFeatureExtractor.class.getName() });

  testClassifier(dataWriter, outDirectoryName, 1, "-t", "0"); // MultiClassLIBSVMClassifier.score
                                                              // is not implemented so we cannot
                                                              // have a stack size greater than 1.
  String firstLine = FileUtil.loadListOfStrings(new File(outDirectoryName
      + "/2008_Sichuan_earthquake.txt.pos"))[0].trim();
  boolean badTags = firstLine.equals("2008/NN Sichuan/NN earthquake/NN From/NN Wikipedia/NN ,/NN the/NN free/NN encyclopedia/NN");
  assertFalse(badTags);
  assertEquals(
      "2008/NN Sichuan/NN earthquake/NN From/IN Wikipedia/NN ,/, the/DT free/NN encyclopedia/NN",
      firstLine);

}
 

import org.cleartk.ml.libsvm.LibSvmStringOutcomeDataWriter; //导入依赖的package包/类
@Test
public void testLibsvm() throws Exception {
  String libsvmDirectoryName = outputDirectory + "/libsvm";
  AnalysisEngineDescription dataWriter = AnalysisEngineFactory.createEngineDescription(
      NonSequenceExamplePosAnnotator.class,
      DefaultDataWriterFactory.PARAM_DATA_WRITER_CLASS_NAME,
      LibSvmStringOutcomeDataWriter.class.getName(),
      DirectoryDataWriterFactory.PARAM_OUTPUT_DIRECTORY,
      libsvmDirectoryName);
  testClassifier(dataWriter, libsvmDirectoryName, "-t", "0", "-c", "0.1");

  String firstLine = FileUtil.loadListOfStrings(new File(libsvmDirectoryName
      + "/2008_Sichuan_earthquake.txt.pos"))[0].trim();
  checkPOS(firstLine);
}
 

import org.cleartk.ml.libsvm.LibSvmStringOutcomeDataWriter; //导入依赖的package包/类
public static void main(String[] args) throws Exception {
  Options options = CliFactory.parseArguments(Options.class, args);

  // ////////////////////////////////////////
  // Create collection reader to load URIs
  // ////////////////////////////////////////
  CollectionReader reader = UriCollectionReader.getCollectionReaderFromDirectory(
      options.getTrainDirectory(),
      UriCollectionReader.RejectSystemFiles.class,
      UriCollectionReader.RejectSystemDirectories.class);

  // ////////////////////////////////////////
  // Create document classification pipeline
  // ////////////////////////////////////////
  AggregateBuilder builder = new AggregateBuilder();

  // Convert URIs in CAS URI View to Plain Text
  builder.add(UriToDocumentTextAnnotator.getDescription());

  // Label documents with gold labels for training
  builder.add(AnalysisEngineFactory.createEngineDescription(GoldDocumentCategoryAnnotator.class));

  // NLP pre-processing components
  builder.add(SentenceAnnotator.getDescription()); // Sentence segmentation
  builder.add(TokenAnnotator.getDescription()); // Tokenization
  builder.add(DefaultSnowballStemmer.getDescription("English")); // Stemming

  // The simple document classification annotator
  builder.add(AnalysisEngineFactory.createEngineDescription(
      BasicDocumentClassificationAnnotator.class,
      DefaultDataWriterFactory.PARAM_DATA_WRITER_CLASS_NAME,
      LibSvmStringOutcomeDataWriter.class.getName(),
      DirectoryDataWriterFactory.PARAM_OUTPUT_DIRECTORY,
      options.getModelsDirectory()));

  // ///////////////////////////////////////////
  // Run pipeline to create training data file
  // ///////////////////////////////////////////
  SimplePipeline.runPipeline(reader, builder.createAggregateDescription());

  // //////////////////////////////////////////////////////////////////////////////
  // Train and write model
  // //////////////////////////////////////////////////////////////////////////////
  JarClassifierBuilder.trainAndPackage(
      options.getModelsDirectory(),
      options.getTrainingArguments().toArray(new String[options.getTrainingArguments().size()]));
}
 

import org.cleartk.ml.libsvm.LibSvmStringOutcomeDataWriter; //导入依赖的package包/类
@Test
public void testMultiClassLIBSVM() throws Exception {
  // create the data writer
  StringAnnotator annotator = new StringAnnotator();
  annotator.initialize(UimaContextFactory.createUimaContext(
      DirectoryDataWriterFactory.PARAM_OUTPUT_DIRECTORY,
      this.outputDirectoryName,
      DefaultDataWriterFactory.PARAM_DATA_WRITER_CLASS_NAME,
      LibSvmStringOutcomeDataWriter.class.getName()));

  // run process to produce a bunch of instances
  annotator.process(null);

  annotator.collectionProcessComplete();

  // check that the output files were written for each class
  BufferedReader reader = new BufferedReader(new FileReader(new File(
      this.outputDirectoryName,
      "training-data.libsvm")));
  Assert.assertTrue(reader.readLine().length() > 0);
  reader.close();

  // run the training command
  HideOutput hider = new HideOutput();
  Train.main(this.outputDirectoryName, "-c", "10", "-t", "0");
  hider.restoreOutput();

  // read in the classifier and test it on new instances
  LibSvmStringOutcomeClassifierBuilder builder = new LibSvmStringOutcomeClassifierBuilder();
  LibSvmStringOutcomeClassifier classifier;
  classifier = builder.loadClassifierFromTrainingDirectory(this.outputDirectory);
  for (Instance<String> instance : ExampleInstanceFactory.generateStringInstances(1000)) {
    List<Feature> features = instance.getFeatures();
    String outcome = instance.getOutcome();
    Assert.assertEquals(outcome, classifier.classify(features));

    Map<String, Double> scoredOutcomes = classifier.score(features);
    for (String otherOutcome : Arrays.asList("A", "B", "C")) {
      if (!otherOutcome.equals(outcome)) {
        Assert.assertTrue(scoredOutcomes.get(outcome) > scoredOutcomes.get(otherOutcome));
      }
    }
  }
}