Java DirectoryDataWriterFactory类代码示例

import org.cleartk.ml.jar.DirectoryDataWriterFactory; //导入依赖的package包/类
/**
 * @param posTagFile
 * @param modelDirectory
 * @param language
 * @throws UIMAException
 * @throws IOException
 */
public static void writeModel(File posTagFile, String modelDirectory, String language) throws UIMAException, IOException {

    CollectionReader posTagFileReader = FilesCollectionReader.getCollectionReaderWithSuffixes(
            posTagFile.getAbsolutePath(), NERReader.CONLL_VIEW, posTagFile.getName());

    AnalysisEngine snowballStemmer = createEngine(SnowballStemmer.class, SnowballStemmer.PARAM_LANGUAGE, language);

    AnalysisEngine nerAnnotator = createEngine(NERAnnotator.class,
            NERAnnotator.PARAM_FEATURE_EXTRACTION_FILE, "src/main/resources/feature/features.xml",
            NERAnnotator.PARAM_IS_TRAINING, true,
            DirectoryDataWriterFactory.PARAM_OUTPUT_DIRECTORY, modelDirectory,
            DefaultDataWriterFactory.PARAM_DATA_WRITER_CLASS_NAME, CrfSuiteStringOutcomeDataWriter.class);

    runPipeline(
            posTagFileReader,
            createEngine(NERReader.class),
            snowballStemmer,
            nerAnnotator
    );
}
 

import org.cleartk.ml.jar.DirectoryDataWriterFactory; //导入依赖的package包/类
/**
 * @param posTagFile
 * @param configFileName
 * @param language
 * @throws UIMAException
 * @throws IOException
 */
private void writeModel(File posTagFile, String language, String configFileName) throws UIMAException, IOException {

    new File(getModelDir()).mkdirs();

    CollectionReader posTagFileReader = FilesCollectionReader.getCollectionReaderWithSuffixes(
            posTagFile.getAbsolutePath(), NERReader.CONLL_VIEW, posTagFile.getName());

    AnalysisEngine snowballStemmer = createEngine(SnowballStemmer.class, SnowballStemmer.PARAM_LANGUAGE, language);

    AnalysisEngine nerAnnotator = createEngine(NERAnnotator.class,
            NERAnnotator.PARAM_FEATURE_EXTRACTION_FILE, FEATURE_EXTRACTOR_CONFIG_DIRECTORY + configFileName,
            NERAnnotator.PARAM_IS_TRAINING, true,
            DirectoryDataWriterFactory.PARAM_OUTPUT_DIRECTORY, getModelDir(),
            DefaultDataWriterFactory.PARAM_DATA_WRITER_CLASS_NAME, CrfSuiteStringOutcomeDataWriter.class);

    runPipeline(
            posTagFileReader,
            createEngine(NERReader.class),
            snowballStemmer,
            nerAnnotator
    );
}
 

import org.cleartk.ml.jar.DirectoryDataWriterFactory; //导入依赖的package包/类
@Override
public void train(CollectionReader collectionReader, File outputDirectory) throws Exception {
  AggregateBuilder builder = new AggregateBuilder();
  builder.add(UriToDocumentTextAnnotator.getDescription());
  builder.add(SentenceAnnotator.getDescription());
  builder.add(TokenAnnotator.getDescription());
  builder.add(PosTaggerAnnotator.getDescription());
  builder.add(DefaultSnowballStemmer.getDescription("English"));
  builder.add(AnalysisEngineFactory.createEngineDescription(GoldQuestionCategoryAnnotator.class));
  AnalysisEngineDescription documentClassificationAnnotator = AnalysisEngineFactory.createEngineDescription(
      QuestionCategoryAnnotator.class, CleartkAnnotator.PARAM_IS_TRAINING, true,
      DirectoryDataWriterFactory.PARAM_OUTPUT_DIRECTORY, outputDirectory,
      DefaultDataWriterFactory.PARAM_DATA_WRITER_CLASS_NAME, LibSvmStringOutcomeDataWriter.class.getName());
  builder.add(documentClassificationAnnotator);
  SimplePipeline.runPipeline(collectionReader, builder.createAggregateDescription());
  System.err.println("Train model and write model.jar file.");
  HideOutput hider = new HideOutput();
  Train.main(outputDirectory, this.trainingArguments.toArray(new String[this.trainingArguments.size()]));
  hider.restoreOutput();
}
 

import org.cleartk.ml.jar.DirectoryDataWriterFactory; //导入依赖的package包/类
@Override
protected void train(CollectionReader collectionReader, File directory) throws Exception {
	String tmpView = "DP";

	AggregateBuilder b = new AggregateBuilder();

	b.add(AnalysisEngineFactory.createEngineDescription(PrepareClearTk.class, PrepareClearTk.PARAM_VIEW_NAME,
			tmpView, PrepareClearTk.PARAM_ANNOTATION_TYPE, DramatisPersonae.class,
			PrepareClearTk.PARAM_SUBANNOTATIONS, Arrays.asList(Figure.class)));
	b.add(AnalysisEngineFactory.createEngineDescription(BreakIteratorSegmenter.class), CAS.NAME_DEFAULT_SOFA,
			tmpView);
	b.add(AnalysisEngineFactory.createEngineDescription(ClearTkGenderAnnotator.class,
			DefaultDataWriterFactory.PARAM_DATA_WRITER_CLASS_NAME, LibSvmStringOutcomeDataWriter.class,
			DirectoryDataWriterFactory.PARAM_OUTPUT_DIRECTORY, directory), CAS.NAME_DEFAULT_SOFA, tmpView);
	b.add(AnalysisEngineFactory.createEngineDescription(XmiWriter.class, XmiWriter.PARAM_TARGET_LOCATION,
			"target/xmi"));

	SimplePipeline.runPipeline(collectionReader, b.createAggregate());

	Train.main(directory, new String[] { "-t", "0" });
}
 

import org.cleartk.ml.jar.DirectoryDataWriterFactory; //导入依赖的package包/类
public static void main(String[] args) throws Exception {
	CollectionReaderDescription reader = CollectionReaderFactory.createReaderDescription(XmiReader.class,
			XmiReader.PARAM_SOURCE_LOCATION, "src/main/resources/gender/*/*.xmi", XmiReader.PARAM_LENIENT, true);

	String tmpView = "DP";

	AggregateBuilder b = new AggregateBuilder();

	b.add(AnalysisEngineFactory.createEngineDescription(PrepareClearTk.class, PrepareClearTk.PARAM_VIEW_NAME,
			tmpView, PrepareClearTk.PARAM_ANNOTATION_TYPE, DramatisPersonae.class,
			PrepareClearTk.PARAM_SUBANNOTATIONS, Arrays.asList(Figure.class)));
	b.add(AnalysisEngineFactory.createEngineDescription(BreakIteratorSegmenter.class), CAS.NAME_DEFAULT_SOFA,
			tmpView);
	b.add(AnalysisEngineFactory.createEngineDescription(ClearTkGenderAnnotator.class,
			DefaultDataWriterFactory.PARAM_DATA_WRITER_CLASS_NAME, LibSvmStringOutcomeDataWriter.class,
			DirectoryDataWriterFactory.PARAM_OUTPUT_DIRECTORY, "target/models"), CAS.NAME_DEFAULT_SOFA, tmpView);
	b.add(AnalysisEngineFactory.createEngineDescription(XmiWriter.class, XmiWriter.PARAM_TARGET_LOCATION,
			"target/xmi"));
	SimplePipeline.runPipeline(reader, b.createAggregateDescription());

	Train.main(new File("target/models"), new String[] { "-t", "0" });
}
 

import org.cleartk.ml.jar.DirectoryDataWriterFactory; //导入依赖的package包/类
public static void main(String[] args) throws Exception {
	Options options = CliFactory.parseArguments(Options.class, args);

	// a reader that loads the URIs of the training files
	CollectionReaderDescription reader = CollectionReaderFactory.createReaderDescription(XmiReader.class,
			XmiReader.PARAM_SOURCE_LOCATION, options.getTrainDirectory() + "/*.xmi", XmiReader.PARAM_LENIENT, true);

	// run the pipeline over the training corpus
	SimplePipeline.runPipeline(reader,
			createEngineDescription(ContextWindowAnnotator.class, ContextWindowAnnotator.PARAM_BASE_ANNOTATION,
					FigureMention.class, ContextWindowAnnotator.PARAM_CONTEXT_CLASS, Speech.class,
					ContextWindowAnnotator.PARAM_TARGET_ANNOTATION, TrainingArea.class),
			createEngineDescription(ClearTkMentionAnnotator.class, CleartkSequenceAnnotator.PARAM_IS_TRAINING, true,
					DirectoryDataWriterFactory.PARAM_OUTPUT_DIRECTORY, options.getModelDirectory(),
					DefaultSequenceDataWriterFactory.PARAM_DATA_WRITER_CLASS_NAME,
					MalletCrfStringOutcomeDataWriter.class),
			createEngineDescription(XmiWriter.class, XmiWriter.PARAM_TARGET_LOCATION, "target/"));

	// train a Mallet CRF model on the training data
	Train.main(options.getModelDirectory());
}
 

import org.cleartk.ml.jar.DirectoryDataWriterFactory; //导入依赖的package包/类
@Test
public void testIssue339() throws Exception {
  AnalysisEngine dataWriterAnnotator = AnalysisEngineFactory.createEngine(
      TestIssue339Annotator.class,
      DefaultDataWriterFactory.PARAM_DATA_WRITER_CLASS_NAME,
      WekaStringOutcomeDataWriter.class.getName(),
      DirectoryDataWriterFactory.PARAM_OUTPUT_DIRECTORY,
      this.outputDirectory);
  dataWriterAnnotator.process(jCas);
  dataWriterAnnotator.collectionProcessComplete();
  File outputFile = new File(this.outputDirectory, "training-data.arff");
  String output = Files.toString(outputFile, Charsets.US_ASCII);
  
  // make sure that at least one instance was written
  Pattern emptyData = Pattern.compile("@data\\s*\\{\\}");
  boolean hasEmptyData = emptyData.matcher(output).find();
  Assert.assertFalse(hasEmptyData);
  
  // make sure that the "NN" value shows up
  Assert.assertTrue(output.contains("0 NN"));
}
 

import org.cleartk.ml.jar.DirectoryDataWriterFactory; //导入依赖的package包/类
@Test
public void testMalletCRF() throws Exception {
  this.assumeLongTestsEnabled();
  this.logger.info(LONG_TEST_MESSAGE);

  String outDirectoryName = outputDirectoryName + "/malletcrf";
  AnalysisEngineDescription dataWriter = AnalysisEngineFactory.createEngineDescription(
      ExamplePosAnnotator.class,
      DefaultSequenceDataWriterFactory.PARAM_DATA_WRITER_CLASS_NAME,
      MalletCrfStringOutcomeDataWriter.class.getName(),
      DirectoryDataWriterFactory.PARAM_OUTPUT_DIRECTORY,
      outDirectoryName);
  testClassifier(dataWriter, outDirectoryName, -1); // viterbi stack size is meaningless here so
                                                    // pass in an invalid value to make sure it is
                                                    // ignored.

  String firstLine = FileUtil.loadListOfStrings(new File(outDirectoryName
      + "/2008_Sichuan_earthquake.txt.pos"))[0].trim();
  assertEquals(
      "2008/NN Sichuan/CD earthquake/NNS From/IN Wikipedia/NN ,/, the/DT free/NN encyclopedia/NN",
      firstLine);

}
 

import org.cleartk.ml.jar.DirectoryDataWriterFactory; //导入依赖的package包/类
@Test
public void testMaxent() throws Exception {
  String outDirectoryName = outputDirectoryName + "/maxent";

  AnalysisEngineDescription dataWriter = AnalysisEngineFactory.createEngineDescription(
      ExamplePosAnnotator.class,
      CleartkSequenceAnnotator.PARAM_DATA_WRITER_FACTORY_CLASS_NAME,
      ViterbiDataWriterFactory.class.getName(),
      DirectoryDataWriterFactory.PARAM_OUTPUT_DIRECTORY,
      outDirectoryName,
      DefaultDataWriterFactory.PARAM_DATA_WRITER_CLASS_NAME,
      MaxentStringOutcomeDataWriter.class.getName(),
      ViterbiDataWriterFactory.PARAM_OUTCOME_FEATURE_EXTRACTOR_NAMES,
      new String[] { DefaultOutcomeFeatureExtractor.class.getName() });
  testClassifier(dataWriter, outDirectoryName, 10);

  String firstLine = FileUtil.loadListOfStrings(new File(outDirectoryName
      + "/2008_Sichuan_earthquake.txt.pos"))[0];
  assertEquals(
      "2008/CD Sichuan/JJ earthquake/NNS From/IN Wikipedia/NN ,/, the/DT free/NN encyclopedia/IN",
      firstLine);
}
 

import org.cleartk.ml.jar.DirectoryDataWriterFactory; //导入依赖的package包/类
@Test
public void testMalletMaxent() throws Exception {
  String outDirectoryName = outputDirectoryName + "/mallet-maxent";

  AnalysisEngineDescription dataWriter = AnalysisEngineFactory.createEngineDescription(
      ExamplePosAnnotator.class,
      CleartkSequenceAnnotator.PARAM_DATA_WRITER_FACTORY_CLASS_NAME,
      ViterbiDataWriterFactory.class.getName(),
      DirectoryDataWriterFactory.PARAM_OUTPUT_DIRECTORY,
      outDirectoryName,
      DefaultDataWriterFactory.PARAM_DATA_WRITER_CLASS_NAME,
      MalletStringOutcomeDataWriter.class.getName(),
      ViterbiDataWriterFactory.PARAM_OUTCOME_FEATURE_EXTRACTOR_NAMES,
      new String[] { DefaultOutcomeFeatureExtractor.class.getName() });
  testClassifier(dataWriter, outDirectoryName, 10, "MaxEnt");

  String firstLine = FileUtil.loadListOfStrings(new File(outDirectoryName
      + "/2008_Sichuan_earthquake.txt.pos"))[0];
  assertEquals(
      "2008/DT Sichuan/JJ earthquake/NN From/IN Wikipedia/NN ,/, the/DT free/NN encyclopedia/NN",
      firstLine);
}
 

import org.cleartk.ml.jar.DirectoryDataWriterFactory; //导入依赖的package包/类
@Test
public void testMalletNaiveBayes() throws Exception {
  String outDirectoryName = outputDirectoryName + "/mallet-naive-bayes";

  AnalysisEngineDescription dataWriter = AnalysisEngineFactory.createEngineDescription(
      ExamplePosAnnotator.class,
      CleartkSequenceAnnotator.PARAM_DATA_WRITER_FACTORY_CLASS_NAME,
      ViterbiDataWriterFactory.class.getName(),
      DirectoryDataWriterFactory.PARAM_OUTPUT_DIRECTORY,
      outDirectoryName,
      DefaultDataWriterFactory.PARAM_DATA_WRITER_CLASS_NAME,
      MalletStringOutcomeDataWriter.class.getName(),
      ViterbiDataWriterFactory.PARAM_OUTCOME_FEATURE_EXTRACTOR_NAMES,
      new String[] { DefaultOutcomeFeatureExtractor.class.getName() });
  testClassifier(dataWriter, outDirectoryName, 10, "NaiveBayes");

  String firstLine = FileUtil.loadListOfStrings(new File(outDirectoryName
      + "/2008_Sichuan_earthquake.txt.pos"))[0];
  assertEquals(
      "2008/DT Sichuan/JJ earthquake/NN From/IN Wikipedia/NN ,/, the/DT free/NN encyclopedia/IN",
      firstLine);
}
 

import org.cleartk.ml.jar.DirectoryDataWriterFactory; //导入依赖的package包/类
@Test
public void testMalletC45() throws Exception {
  this.assumeLongTestsEnabled();
  this.logger.info(LONG_TEST_MESSAGE);

  String outDirectoryName = outputDirectoryName + "/mallet-c45";

  AnalysisEngineDescription dataWriter = AnalysisEngineFactory.createEngineDescription(
      ExamplePosAnnotator.class,
      CleartkSequenceAnnotator.PARAM_DATA_WRITER_FACTORY_CLASS_NAME,
      ViterbiDataWriterFactory.class.getName(),
      DirectoryDataWriterFactory.PARAM_OUTPUT_DIRECTORY,
      outDirectoryName,
      DefaultDataWriterFactory.PARAM_DATA_WRITER_CLASS_NAME,
      MalletStringOutcomeDataWriter.class.getName(),
      ViterbiDataWriterFactory.PARAM_OUTCOME_FEATURE_EXTRACTOR_NAMES,
      new String[] { DefaultOutcomeFeatureExtractor.class.getName() });
  testClassifier(dataWriter, outDirectoryName, 10, "C45");

  String firstLine = FileUtil.loadListOfStrings(new File(outDirectoryName
      + "/2008_Sichuan_earthquake.txt.pos"))[0];
  assertEquals(
      "2008/CD Sichuan/JJ earthquake/NN From/NN Wikipedia/NN ,/, the/DT free/NN encyclopedia/NN",
      firstLine);
}
 

import org.cleartk.ml.jar.DirectoryDataWriterFactory; //导入依赖的package包/类
@Test
public void testMaxent() throws Exception {
  String maxentDirectoryName = outputDirectoryName + "/maxent";
  AnalysisEngineDescription dataWriter = AnalysisEngineFactory.createEngineDescription(
      NonSequenceExamplePosAnnotator.class,
      DefaultDataWriterFactory.PARAM_DATA_WRITER_CLASS_NAME,
      MaxentStringOutcomeDataWriter.class.getName(),
      DirectoryDataWriterFactory.PARAM_OUTPUT_DIRECTORY,
      maxentDirectoryName);
  testClassifier(dataWriter, maxentDirectoryName);

  // Not sure why the _SPLIT is here, but we will throw it out for good measure
  String firstLine = FileUtil.loadListOfStrings(new File(maxentDirectoryName
      + "/2008_Sichuan_earthquake.txt.pos"))[0].trim().replace("_SPLIT", "");
  checkPOS(firstLine);
}
 

import org.cleartk.ml.jar.DirectoryDataWriterFactory; //导入依赖的package包/类
@Test
public void testSVMLIGHT() throws Exception {
  this.assumeTestsEnabled(ExamplePosClassifierTest.SVMLIGHT_TESTS_PROPERTY_VALUE);
  this.logger.info(ExamplePosClassifierTest.SVMLIGHT_TESTS_ENABLED_MESSAGE);

  String svmlightDirectoryName = outputDirectoryName + "/svmlight";
  AnalysisEngineDescription dataWriter = AnalysisEngineFactory.createEngineDescription(
      NonSequenceExamplePosAnnotator.class,
      DefaultDataWriterFactory.PARAM_DATA_WRITER_CLASS_NAME,
      SvmLightStringOutcomeDataWriter.class.getName(),
      DirectoryDataWriterFactory.PARAM_OUTPUT_DIRECTORY,
      svmlightDirectoryName);
  testClassifier(dataWriter, svmlightDirectoryName, "-c", "0.1");

  String firstLine = FileUtil.loadListOfStrings(new File(svmlightDirectoryName
      + "/2008_Sichuan_earthquake.txt.pos"))[0].trim();
  checkPOS(firstLine);
}
 

import org.cleartk.ml.jar.DirectoryDataWriterFactory; //导入依赖的package包/类
public AnalysisEngineDescription getWriterDescription(File outputDirectory)
    throws ResourceInitializationException {
  Class<?> dataWriterClass = this.getDataWriterClass();
  String paramName;
  if (SequenceDataWriter.class.isAssignableFrom(dataWriterClass)) {
    paramName = DefaultSequenceDataWriterFactory.PARAM_DATA_WRITER_CLASS_NAME;
  } else if (DataWriter.class.isAssignableFrom(dataWriterClass)) {
    paramName = DefaultDataWriterFactory.PARAM_DATA_WRITER_CLASS_NAME;
  } else {
    throw new RuntimeException("Invalid data writer class: " + dataWriterClass);
  }
  AnalysisEngineDescription desc = getBaseDescription();
  ResourceCreationSpecifierFactory.setConfigurationParameters(
      desc,
      DirectoryDataWriterFactory.PARAM_OUTPUT_DIRECTORY,
      outputDirectory.getPath(),
      paramName,
      dataWriterClass.getName());
  return desc;
}