Java ChunkerModel类代码示例

import opennlp.tools.chunker.ChunkerModel; //导入依赖的package包/类
public static void main(String[] args) throws IOException {
    if (args.length == 0) {
        System.err.println("Usage: AnswerTypeEventStream eventfile");
        System.exit(1);
    }
    int ai = 0;
    String eventFile = args[ai++];
    String modelsDirProp = System.getProperty("models.dir", "book/src/main" + File.separator + "opennlp-models" +
            File.separator + "english");
    File modelsDir = new File(modelsDirProp);
    File wordnetDir = new File(System.getProperty("wordnet.dir", "book/src/main" + File.separator + "WordNet-3.0" + File.separator + "dict"));
    InputStream chunkerStream = new FileInputStream(
            new File(modelsDir, "en-chunker.bin"));
    ChunkerModel chunkerModel = new ChunkerModel(chunkerStream);
    ChunkerME chunker = new ChunkerME(chunkerModel);
    InputStream posStream = new FileInputStream(
            new File(modelsDir, "en-pos-maxent.bin"));
    POSModel posModel = new POSModel(posStream);
    POSTaggerME tagger = new POSTaggerME(posModel);
    Parser parser = new ChunkParser(chunker, tagger);
    AnswerTypeContextGenerator actg = new AnswerTypeContextGenerator(wordnetDir);
    EventStream es = new AnswerTypeEventStream(eventFile, actg, parser);
    while (es.hasNext()) {
        System.out.println(es.next().toString());
    }
}
 

import opennlp.tools.chunker.ChunkerModel; //导入依赖的package包/类
public static void trainChunker(final String inResource, String outFile) throws IOException {
    InputStreamFactory inputStreamFactory = new InputStreamFactory() {
        @Override
        public InputStream createInputStream() throws IOException {
            return Trainer.class.getResourceAsStream(inResource);
        }
    };
    ChunkSampleStream samples = new ChunkSampleStream(new PlainTextByLineStream(inputStreamFactory, StandardCharsets.UTF_8));
    TrainingParameters trainingParameters = new TrainingParameters();
    trainingParameters.put(TrainingParameters.ITERATIONS_PARAM, "70");
    trainingParameters.put(TrainingParameters.CUTOFF_PARAM, "1");

    ChunkerFactory chunkerFactory = ChunkerFactory.create(null);
    ChunkerModel model = ChunkerME.train("en", samples, trainingParameters, chunkerFactory);
    //ChunkerME.train("en", samples, 1, 70);
    samples.close();
    FileOutputStream out = new FileOutputStream(outFile);
    model.serialize(out);
    out.close();
}
 

import opennlp.tools.chunker.ChunkerModel; //导入依赖的package包/类
@Override
public void doInitialize(UimaContext aContext) throws ResourceInitializationException {
	try {
		tokensModel.loadModel(TokenizerModel.class, getClass().getResourceAsStream("en_token.bin"));
		sentencesModel.loadModel(SentenceModel.class, getClass().getResourceAsStream("en_sent.bin"));
		posModel.loadModel(POSModel.class, getClass().getResourceAsStream("en_pos_maxent.bin"));
		chunkModel.loadModel(ChunkerModel.class, getClass().getResourceAsStream("en_chunker.bin"));
	} catch (BaleenException be) {
		getMonitor().error("Unable to load OpenNLP Language Models", be);
		throw new ResourceInitializationException(be);
	}

	try {
		sentenceDetector = new SentenceDetectorME((SentenceModel) sentencesModel.getModel());
		wordTokenizer = new TokenizerME((TokenizerModel) tokensModel.getModel());
		posTagger = new POSTaggerME((POSModel) posModel.getModel());
		phraseChunker = new ChunkerME((ChunkerModel) chunkModel.getModel());
	} catch (Exception e) {
		getMonitor().error("Unable to create OpenNLP taggers", e);
		throw new ResourceInitializationException(e);
	}
}
 

import opennlp.tools.chunker.ChunkerModel; //导入依赖的package包/类
@Test
public void testChunker(){
	try (InputStream modelIn = BasicActions.class.getClassLoader().
				getResourceAsStream(Consts.EN_CHUNK_MODEL);){
		
		String[] tokens = testTokenizer();
		String[] tags = testTagger();
		
		ChunkerModel chunkerModel = new ChunkerModel(modelIn);
		ChunkerME chunker = new ChunkerME(chunkerModel);
		String chunks[] = chunker.chunk(tokens, tags);
			System.out.println(Arrays.toString(chunks));
	} catch (IOException e) {
		e.printStackTrace();
	}
}
 

import opennlp.tools.chunker.ChunkerModel; //导入依赖的package包/类
public KeyPhraseChunkExtractor() throws Exception, IOException {

		InputStream modelIn = getClass().getResourceAsStream(
				"/nlptools/data/en-pos-maxent.bin");
		posModel = new POSModel(modelIn);
		tagger = new POSTaggerME(posModel);

		modelIn = getClass().getResourceAsStream(
				"/nlptools/data/en-chunker.bin");
		chunkModel = new ChunkerModel(modelIn);
		chunker = new ChunkerME(chunkModel);

		modelIn = getClass().getResourceAsStream("/nlptools/data/en-token.bin");
		nlTokenizerModel = new TokenizerModel(modelIn);
		nlTokenizer = new TokenizerME(nlTokenizerModel);
	}
 

import opennlp.tools.chunker.ChunkerModel; //导入依赖的package包/类
/**
 * Main entry point for training.
 * 
 * @throws IOException
 *           throws an exception if errors in the various file inputs.
 */
public final void train() throws IOException {
  // load training parameters file
  final String paramFile = this.parsedArguments.getString("params");
  final TrainingParameters params = InputOutputUtils
      .loadTrainingParameters(paramFile);
  String outModel = null;
  if (params.getSettings().get("OutputModel") == null
      || params.getSettings().get("OutputModel").length() == 0) {
    outModel = Files.getNameWithoutExtension(paramFile) + ".bin";
    params.put("OutputModel", outModel);
  } else {
    outModel = Flags.getModel(params);
  }
  final Trainer chunkerTrainer = new DefaultTrainer(params);
  final ChunkerModel trainedModel = chunkerTrainer.train(params);
  CmdLineUtil.writeModel("ixa-pipe-chunk", new File(outModel), trainedModel);
}
 

import opennlp.tools.chunker.ChunkerModel; //导入依赖的package包/类
public final ChunkerModel train(final TrainingParameters params) {
  // features
  if (getChunkerFactory() == null) {
    throw new IllegalStateException(
        "Classes derived from AbstractTrainer must "
            + " create a ChunkerFactory features!");
  }
  // training model
  ChunkerModel trainedModel = null;
  ChunkerEvaluator chunkerEvaluator = null;
  try {
    trainedModel = ChunkerME.train(lang, trainSamples, params,
        getChunkerFactory());
    final Chunker chunker = new ChunkerME(trainedModel);
    chunkerEvaluator = new ChunkerEvaluator(chunker);
    chunkerEvaluator.evaluate(this.testSamples);
  } catch (IOException e) {
    System.err.println("IO error while loading traing and test sets!");
    e.printStackTrace();
    System.exit(1);
  }
  System.out.println("Final result: " + chunkerEvaluator.getFMeasure());
  return trainedModel;
}
 

import opennlp.tools.chunker.ChunkerModel; //导入依赖的package包/类
/**
 * Loads statically the probabilistic model. Every instance of this finder
 * will share the same model.
 * 
 * @param lang
 *          the language
 * @param model
 *          the model to be loaded
 * @return the model as a {@link ChunkerModel} object
 */
private ChunkerModel loadModel(final String lang, final String model) {
  final long lStartTime = new Date().getTime();
  try {
    synchronized (chunkerModels) {
      if (!chunkerModels.containsKey(lang)) {
        chunkerModels.put(lang, new ChunkerModel(new FileInputStream(model)));
      }
    }
  } catch (final IOException e) {
    e.printStackTrace();
  }
  final long lEndTime = new Date().getTime();
  final long difference = lEndTime - lStartTime;
  System.err.println("ixa-pipe-chunk model loaded in: " + difference
      + " miliseconds ... [DONE]");
  return chunkerModels.get(lang);
}
 

import opennlp.tools.chunker.ChunkerModel; //导入依赖的package包/类
@SuppressWarnings("rawtypes")
public void init(NamedList initArgs) {
	
    SolrParams params = SolrParams.toSolrParams(initArgs);
    String modelDirectory = params.get("modelDirectory",
            System.getProperty("model.dir"));//<co id="qqpp.model"/>
    String wordnetDirectory = params.get("wordnetDirectory",
            System.getProperty("wordnet.dir"));//<co id="qqpp.wordnet"/>
    if (modelDirectory != null) {
      File modelsDir = new File(modelDirectory);
      try {
        InputStream chunkerStream = new FileInputStream(
            new File(modelsDir,"en-chunker.bin"));
        ChunkerModel chunkerModel = new ChunkerModel(chunkerStream);
        chunker = new ChunkerME(chunkerModel); //<co id="qqpp.chunker"/>
        InputStream posStream = new FileInputStream(
            new File(modelsDir,"en-pos-maxent.bin"));
        POSModel posModel = new POSModel(posStream);
        tagger =  new POSTaggerME(posModel); //<co id="qqpp.tagger"/>
       // model = new DoccatModel(new FileInputStream( //<co id="qqpp.theModel"/>
     //       new File(modelDirectory,"en-answer.bin"))).getMaxentModel();
        model = new SuffixSensitiveGISModelReader(new File(modelDirectory+"/qa/ans.bin")).getModel();
        //GISModel m = new SuffixSensitiveGISModelReader(new File(modelFileName)).getModel(); 
        probs = new double[model.getNumOutcomes()];
        atcg = new AnswerTypeContextGenerator(
                new File(wordnetDirectory, "dict"));//<co id="qqpp.context"/>
      } catch (IOException e) {
        throw new RuntimeException(e);
      }
    }
  }
 

import opennlp.tools.chunker.ChunkerModel; //导入依赖的package包/类
/**
 * Train the answer model
 * <p>
 * Hint:
 * <pre>
 *  mvn exec:java -Dexec.mainClass=com.tamingtext.qa.AnswerTypeClassifier \
 *    -Dexec.args="dist/data/questions-train.txt en-answer.bin" \
 *    -Dmodel.dir=../../opennlp-models \
 *    -Dwordnet.dir=../../Wordnet-3.0/dict
 *  </pre>
 *
 * @param args
 * @throws IOException
 */
public static void main(String[] args) throws IOException {
    if (args.length < 2) {
        System.err.println("Usage: AnswerTypeClassifier <trainFile> <modelFile>");
        System.exit(1);
    }

    String trainFile = args[0];
    File outFile = new File(args[1]);
    String modelsDirProp = System.getProperty("model.dir");
    File modelsDir = new File(modelsDirProp);
    String wordnetDir = System.getProperty("wordnet.dir");

    InputStream chunkerStream = new FileInputStream(
            new File(modelsDir, "en-chunker.bin"));
    ChunkerModel chunkerModel = new ChunkerModel(chunkerStream);
    ChunkerME chunker = new ChunkerME(chunkerModel);
    InputStream posStream = new FileInputStream(
            new File(modelsDir, "en-pos-maxent.bin"));
    POSModel posModel = new POSModel(posStream);
    POSTaggerME tagger = new POSTaggerME(posModel);
    Parser parser = new ChunkParser(chunker, tagger);
    AnswerTypeContextGenerator actg = new AnswerTypeContextGenerator(new File(wordnetDir));
    //<start id="atc.train"/>
    AnswerTypeEventStream es = new AnswerTypeEventStream(trainFile,
            actg, parser);
    GISModel model = GIS.trainModel(100, new TwoPassDataIndexer(es, 3));//<co id="atc.train.do"/>
    GISModelWriter writer = new SuffixSensitiveGISModelWriter(model, outFile);
    writer.persist();
    //new DoccatModel("en", model).serialize(new FileOutputStream(outFile));
/*
<calloutlist>
    <callout arearefs="atc.train.do"><para>Using the event stream, which feeds us training examples, do the actual training using OpenNLP's Maxent classifier.</para></callout>
</calloutlist>
*/
    //<end id="atc.train"/>
}
 

import opennlp.tools.chunker.ChunkerModel; //导入依赖的package包/类
public static void main(String args[]) throws IOException
{
	String wordnetDir = System.getProperty("wordnet.dir");
	//wordnetDir="WordNet-3.0/dict/";
	String question="Who is Abraham Lincoln?";
	AnswerTypeContextGenerator atcg=new AnswerTypeContextGenerator(new File(wordnetDir));
	String q=null;
    String modelsDirProp = System.getProperty("model.dir");
   // modelsDirProp="opennlp-models/";
    File modelsDir = new File(modelsDirProp);
    InputStream chunkerStream = new FileInputStream(
        new File(modelsDir,"en-chunker.bin"));
    ChunkerModel chunkerModel = new ChunkerModel(chunkerStream);
    ChunkerME chunker = new ChunkerME(chunkerModel);
    InputStream posStream = new FileInputStream(
        new File(modelsDir,"en-pos-maxent.bin"));
    POSModel posModel = new POSModel(posStream);
    POSTaggerME tagger =  new POSTaggerME(posModel);
    Parser parser = new ChunkParser(chunker, tagger);
    
    Parse query = ParserTool.parseLine(question,parser,1)[0];
	String[] context=atcg.getContext(query);
	for(int i=0;i<context.length;i++)
	{
		if(context[i].startsWith("hw=") || context[i].startsWith("mw="))
		{
			System.out.println(context[i].substring(3));
		}
	}
}
 

import opennlp.tools.chunker.ChunkerModel; //导入依赖的package包/类
public String[] getFocusNoun(String question) throws IOException
{
	String wordnetDir = System.getProperty("wordnet.dir");
	wordnetDir="WordNet-3.0/dict/";
	AnswerTypeContextGenerator atcg=new AnswerTypeContextGenerator(new File(wordnetDir));
	String q=null;
    String modelsDirProp = System.getProperty("model.dir");
    modelsDirProp="opennlp-models/";
    File modelsDir = new File(modelsDirProp);
    InputStream chunkerStream = new FileInputStream(
        new File(modelsDir,"en-chunker.bin"));
    ChunkerModel chunkerModel = new ChunkerModel(chunkerStream);
    ChunkerME chunker = new ChunkerME(chunkerModel);
    InputStream posStream = new FileInputStream(
        new File(modelsDir,"en-pos-maxent.bin"));
    POSModel posModel = new POSModel(posStream);
    POSTaggerME tagger =  new POSTaggerME(posModel);
    Parser parser = new ChunkParser(chunker, tagger);
    
    Parse query = ParserTool.parseLine(question,parser,1)[0];
	String[] context=atcg.getContext(query);
	String[] focus=new String[2];
	int p=0;
	for(int i=0;i<context.length;i++)
	{
		if(context[i].startsWith("hw=") || context[i].startsWith("mw="))
		{
			//System.out.println(context[i].substring(3));
			focus[p++]=context[i].substring(3);
		}
	}
	return focus;
}
 

import opennlp.tools.chunker.ChunkerModel; //导入依赖的package包/类
public EnglishIndexer() throws Exception {
      mDicts = new EnglishDictionaries();
      mBeamSize = ConfigurationManager.getConfiguration().getInt("BeamSize");
      InputStream modelStream = null;

      modelStream = getClass().getResourceAsStream("/opennlp15model-sa/en-sent.bin");
SentenceModel model = new SentenceModel(modelStream);
mSentenceDetector = new SentenceDetectorME(model);
modelStream.close();

modelStream = getClass().getResourceAsStream("/opennlp15model-sa/en-token.bin");
mTokenizer = new EnglishTokenizer(modelStream, mDicts);
modelStream.close();

// The parser model is about 15x the size of chunking model. 
// Keep this in mind when using Deep Parsing.
modelStream = getClass().getResourceAsStream("/opennlp15model-sa/en-pos-maxent.bin");
//POSModel posModel = POSTaggerUtils.createPOSModel(modelStream);
POSModel posModel = new POSModel(modelStream);
mTagDictionary = posModel.getTagDictionary();
mPosTagger = new POSTaggerME(posModel);
modelStream.close();

modelStream = getClass().getResourceAsStream("/opennlp15model-sa/en-chunker.bin");
ChunkerModel chunkerModel = new ChunkerModel(modelStream);
mChunker = new ChunkerME(chunkerModel);
modelStream.close();

modelStream = getClass().getResourceAsStream("/opennlp15model-sa/en-parser-chunking.bin");
ParserModel parserModel = new ParserModel(modelStream);
mParser = ParserFactory.create(parserModel);
modelStream.close();

  }
 

import opennlp.tools.chunker.ChunkerModel; //导入依赖的package包/类
public Chunker(Double threshold) {
    this.threshold = threshold;
    try (InputStream modelStream = Chunker.class.getResourceAsStream(MODEL)) {
        ChunkerModel chunkerModel = new ChunkerModel(modelStream);
        chunker = new ChunkerME(chunkerModel);
    } catch (IOException e) {
        LOGGER.error("an error occurred while getting chunks", e);
        throw new IllegalStateException(e);
    }
}
 

import opennlp.tools.chunker.ChunkerModel; //导入依赖的package包/类
@Override
public boolean initialize(ResourceSpecifier aSpecifier, Map<String, Object> aAdditionalParams)
        throws ResourceInitializationException {
  boolean ret = super.initialize(aSpecifier, aAdditionalParams);
  String model = String.class.cast(getParameterValue("chunker-model"));
  try (InputStream ois = getClass().getResourceAsStream(model)) {
    chunker = new ChunkerME(new ChunkerModel(ois));
    Streams.closeQuietly(ois);
  } catch (Exception e) {
    throw new ResourceInitializationException(e);
  }
  type = Arrays.asList(String.class.cast(getParameterValue("type")).split(","));
  minLength = Integer.class.cast(getParameterValue("min-length"));
  return ret;
}