Java MaxentModel类代码示例

import opennlp.tools.ml.model.MaxentModel; //导入依赖的package包/类
public static AgeClassifyModel train(String languageCode,
       ObjectStream<AuthorAgeSample> samples, TrainingParameters trainParams,
      	AgeClassifyFactory factory) throws IOException {

Map<String, String> entries = new HashMap<String, String>();

MaxentModel ageModel = null;

TrainerType trainerType = AgeClassifyTrainerFactory
    .getTrainerType(trainParams.getSettings());

ObjectStream<Event> eventStream = new AgeClassifyEventStream(samples,
    factory.createContextGenerator());

EventTrainer trainer = AgeClassifyTrainerFactory
    .getEventTrainer(trainParams.getSettings(), entries);
ageModel = trainer.train(eventStream);

Map<String, String> manifestInfoEntries = new HashMap<String, String>();

return new AgeClassifyModel(languageCode, ageModel, manifestInfoEntries,
			    factory);
   }
 

import opennlp.tools.ml.model.MaxentModel; //导入依赖的package包/类
@SuppressWarnings("unchecked")
@Override
public void initialize(UimaContext ctx) throws ResourceInitializationException {
    super.initialize(ctx);
    //
    modelAggregate = modelAggregateHolder.getModel();
    //
    if (sequenceValidatorClassName != null) {
        sequenceValidator = InitializableFactory.create(ctx, sequenceValidatorClassName,
                SequenceValidator.class);
    }
    //
    POSTaggerFactory factory = modelAggregate.getFactory();
    MaxentModel posModel = modelAggregate.getPosModel();
    BeamSearchContextGenerator<Token> contextGen = factory.getContextGenerator();
    beam = new BeamSearch<>(beamSize, contextGen, posModel, sequenceValidator, 0);
}
 

import opennlp.tools.ml.model.MaxentModel; //导入依赖的package包/类
/**
 * Initializes the current instance with the provided model and the default
 * beam size of 3.
 * 
 * @param model
 *          the model
 */
public LemmatizerME(LemmatizerModel model) {

  LemmatizerFactory factory = model.getFactory();
  int defaultBeamSize = LemmatizerME.DEFAULT_BEAM_SIZE;
  String beamSizeString = model
      .getManifestProperty(BeamSearch.BEAM_SIZE_PARAMETER);
  if (beamSizeString != null) {
    defaultBeamSize = Integer.parseInt(beamSizeString);
  }

  contextGenerator = factory.getContextGenerator();
  beamSize = defaultBeamSize;

  sequenceValidator = factory.getSequenceValidator();

  if (model.getLemmatizerSequenceModel() != null) {
    this.model = model.getLemmatizerSequenceModel();
  } else {
    this.model = new opennlp.tools.ml.BeamSearch<String>(beamSize,
        (MaxentModel) model.getLemmatizerSequenceModel(), 0);
  }
}
 

import opennlp.tools.ml.model.MaxentModel; //导入依赖的package包/类
public SequenceClassificationModel<String> getLemmatizerSequenceModel() {

    Properties manifest = (Properties) artifactMap.get(MANIFEST_ENTRY);

    if (artifactMap.get(LEMMATIZER_MODEL_ENTRY_NAME) instanceof MaxentModel) {
      String beamSizeString = manifest.getProperty(BeamSearch.BEAM_SIZE_PARAMETER);

      int beamSize = LemmatizerME.DEFAULT_BEAM_SIZE;
      if (beamSizeString != null) {
        beamSize = Integer.parseInt(beamSizeString);
      }

      return new BeamSearch<>(beamSize, (MaxentModel) artifactMap.get(LEMMATIZER_MODEL_ENTRY_NAME));
    }
    else if (artifactMap.get(LEMMATIZER_MODEL_ENTRY_NAME) instanceof SequenceClassificationModel) {
      return (SequenceClassificationModel) artifactMap.get(LEMMATIZER_MODEL_ENTRY_NAME);
    }
    else {
      return null;
    }
  }
 

import opennlp.tools.ml.model.MaxentModel; //导入依赖的package包/类
public ProfilerModel(String languageCode, MaxentModel profilerModel,
  Map<String, String> manifestInfoEntries, ProfilerFactory factory) {
  super(COMPONENT_NAME, languageCode, manifestInfoEntries, factory);

  artifactMap.put(PROFILER_MODEL_ENTRY_NAME, profilerModel);
  checkArtifactMap();
}
 

import opennlp.tools.ml.model.MaxentModel; //导入依赖的package包/类
public POSModel(String languageCode, MaxentModel posModel,
                Map<String, String> manifestInfoEntries, POSTaggerFactory posFactory) {

    super(COMPONENT_NAME, languageCode, manifestInfoEntries, posFactory);

    if (posModel == null)
        throw new IllegalArgumentException("The maxentPosModel param must not be null!");

    artifactMap.put(POS_MODEL_ENTRY_NAME, posModel);
    checkArtifactMap();
}
 

import opennlp.tools.ml.model.MaxentModel; //导入依赖的package包/类
@Override
protected void validateArtifactMap() throws InvalidFormatException {
    super.validateArtifactMap();

    if (!(artifactMap.get(POS_MODEL_ENTRY_NAME) instanceof MaxentModel)) {
        throw new InvalidFormatException("POS model is incomplete!");
    }
}
 

import opennlp.tools.ml.model.MaxentModel; //导入依赖的package包/类
public void train() throws IOException {
    if (languageCode == null) {
        throw new IllegalStateException("languageCode is not provided");
    }
    if (modelOutFile == null) {
        throw new IllegalStateException("model output path is not provided");
    }
    if (trainParams == null) {
        throw new IllegalStateException("training parameters are not set");
    }
    if (sentenceStream == null) {
        throw new IllegalStateException("sentence stream is not configured");
    }
    if (taggerFactory == null) {
        throw new IllegalStateException("tagger factory is not configured");
    }
    Map<String, String> manifestInfoEntries = new HashMap<>();
    BeamSearchContextGenerator<Token> contextGenerator = taggerFactory.getContextGenerator();

    MaxentModel posModel;
    try {
        if (TrainerFactory.TrainerType.EVENT_MODEL_TRAINER.equals(
                TrainerFactory.getTrainerType(trainParams.getSettings()))) {

            ObjectStream<Event> es = new POSTokenEventStream<>(sentenceStream, contextGenerator);
            EventTrainer trainer = TrainerFactory.getEventTrainer(trainParams.getSettings(), manifestInfoEntries);
            posModel = trainer.train(es);
        } else {
            throw new UnsupportedOperationException("Sequence training");
            //POSSampleSequenceStream ss = new POSSampleSequenceStream(samples, contextGenerator);
            // posModel = TrainUtil.train(ss, trainParams.getSettings(), manifestInfoEntries);
        }
    } finally {
        sentenceStream.close();
    }
    POSModel modelAggregate = new POSModel(languageCode,
            posModel, manifestInfoEntries, taggerFactory);
    CmdLineUtil.writeModel("PoS-tagger", modelOutFile, modelAggregate);
}
 

import opennlp.tools.ml.model.MaxentModel; //导入依赖的package包/类
public LemmatizerModel(String languageCode, MaxentModel lemmatizerModel, int beamSize,
    Map<String, String> manifestInfoEntries, LemmatizerFactory factory) {
  super(COMPONENT_NAME, languageCode, manifestInfoEntries, factory);
  artifactMap.put(LEMMATIZER_MODEL_ENTRY_NAME, lemmatizerModel);

  Properties manifest = (Properties) artifactMap.get(MANIFEST_ENTRY);
  manifest.put(BeamSearch.BEAM_SIZE_PARAMETER, Integer.toString(beamSize));
  checkArtifactMap();
}
 

import opennlp.tools.ml.model.MaxentModel; //导入依赖的package包/类
@Override
public MaxentModel doTrain(DataIndexer indexer) throws IOException {

  List<Double> vy = new ArrayList<Double>();
  List<Feature[]> vx = new ArrayList<Feature[]>();

  // outcomes
  int outcomes[] = indexer.getOutcomeList();

  int max_index = 0;
  
  // For each event ...
  for (int i = 0; i < indexer.getContexts().length; i++) {

    int outcome = outcomes[i];
    vy.add(Double.valueOf(outcome));

    int features[] = indexer.getContexts()[i];

    Feature[] x;
    if (bias >= 0) {
      x = new Feature[features.length + 1];
    } else {
      x = new Feature[features.length];
    }

    // for each feature ...
    for (int fi = 0; fi < features.length; fi++) {
      // TODO: SHOUDL BE indexer.getNumTimesEventsSeen()[i] and not fi !!!
      x[fi] = new FeatureNode(features[fi] + 1, indexer.getNumTimesEventsSeen()[i]);
    } 

    if (features.length > 0) {
      max_index = Math.max(max_index, x[features.length - 1].getIndex());
    }
    
    vx.add(x);
  }

  Problem problem = constructProblem(vy, vx, max_index, bias);
  Parameter parameter = new Parameter(solverType, c, eps, p);
  
  Model liblinearModel = Linear.train(problem, parameter);

  Map<String, Integer> predMap = new HashMap<String, Integer>();
  
  String predLabels[] = indexer.getPredLabels();
  for (int i = 0; i < predLabels.length; i++) {
    predMap.put(predLabels[i], i);
  }
  
  return new LiblinearModel(liblinearModel, indexer.getOutcomeLabels(), predMap);
}
 

import opennlp.tools.ml.model.MaxentModel; //导入依赖的package包/类
public static AgeClassifyModel createModel(String languageCode, String dataIn,
				       String tokenizer, String featureGenerators,
				       TrainingParameters trainParams) throws IOException {

SparkConf conf = new SparkConf().setAppName("AgeClassifySparkTrainer");
JavaSparkContext sc = new JavaSparkContext(conf);

AgeClassifyContextGeneratorWrapper wrapper = new AgeClassifyContextGeneratorWrapper(
    tokenizer, featureGenerators);

JavaRDD<String> data = sc.textFile(dataIn, 8).cache();
JavaRDD<EventWrapper> samples = data.map(new CreateEvents(wrapper)).cache();

/*
JavaRDD<EventWrapper> samples = data.map( 
     new Function<String, EventWrapper>() {
	 public EventWrapper call(String s) {
	     String[] parts = s.split(",");
	     
	     try {
		 if (parts[0] != "-1") {
		     Integer value = Integer.parseInt(parts[0]);
		     
		     String[] text = parts[2].split(" ");
		     return new EventWrapper(value, text);
		 } else {
		     String cat = parts[1];
		     
		     String[] text = parts[2].split(" ");
		     return new EventWrapper(cat, text);
		 }
	     } catch(Exception e) {
		 return null;
	     }
	 }
     });
*/

JavaRDD<EventWrapper> validSamples = samples.filter(
           new Function<EventWrapper, Boolean>() {
	@Override
	public Boolean call(EventWrapper s) { return s != null; }
    }).cache();

//ObjectStream<Event> eventStream = EventStreamUtil.createEventStream(samples);	
ObjectStream<Event> eventStream = EventStreamUtil.createEventStream(validSamples.collect());

Map<String, String> entries = new HashMap<String, String>();
				       					    
EventTrainer trainer = AgeClassifyTrainerFactory
           .getEventTrainer(trainParams.getSettings(), entries);
       MaxentModel ageModel = trainer.train(eventStream);

samples.unpersist();
data.unpersist();

sc.stop();

Map<String, String> manifestInfoEntries = new HashMap<String, String>();

AgeClassifyFactory factory = AgeClassifyFactory.create("AgeClassifyFactory", 
           wrapper.getTokenizer(), wrapper.getFeatureGenerators());
return new AgeClassifyModel(languageCode, ageModel, manifestInfoEntries, factory);
   }
 

import opennlp.tools.ml.model.MaxentModel; //导入依赖的package包/类
public AgeClassifyModel(String languageCode, MaxentModel ageClassifyModel,
		    Map<String, String> manifestInfoEntries, AgeClassifyFactory factory) {
super(COMPONENT_NAME, languageCode, manifestInfoEntries, factory);
artifactMap.put(AUTHORAGE_MODEL_ENTRY_NAME, ageClassifyModel);
checkArtifactMap();
   }
 

import opennlp.tools.ml.model.MaxentModel; //导入依赖的package包/类
public MaxentModel getMaxentModel() {
return (MaxentModel) artifactMap.get(AUTHORAGE_MODEL_ENTRY_NAME);
   }
 

import opennlp.tools.ml.model.MaxentModel; //导入依赖的package包/类
public MaxentModel getMaxentModel() {
  return (MaxentModel) artifactMap.get(PROFILER_MODEL_ENTRY_NAME);
}
 

import opennlp.tools.ml.model.MaxentModel; //导入依赖的package包/类
public MaxentModel getPosModel() {
    return (MaxentModel) artifactMap.get(POS_MODEL_ENTRY_NAME);
}