Java DataIndexer类代码示例

import opennlp.tools.ml.model.DataIndexer; //导入依赖的package包/类
@Override
   public DataIndexer getDataIndexer(ObjectStream<Event> events) throws IOException {

String dataIndexerName = getStringParam(DATA_INDEXER_PARAM,
					DATA_INDEXER_TWO_PASS_VALUE);

int cutoff = getCutoff();
boolean sortAndMerge = isSortAndMerge();
DataIndexer indexer = null;

if (DATA_INDEXER_ONE_PASS_VALUE.equals(dataIndexerName)) {
    indexer = new OnePassDataIndexer(events, cutoff, sortAndMerge);
} else if (DATA_INDEXER_TWO_PASS_VALUE.equals(dataIndexerName)) {
    indexer = new TwoPassDataIndexer(events, cutoff, sortAndMerge);
} else if (DATA_INDEXER_CHI_SQUARED.equals(dataIndexerName)) {
    indexer = new ChiSquaredDataIndexer(events, cutoff, sortAndMerge);
}
else {
    throw new IllegalStateException("Unexpected data indexer name: "
				    + dataIndexerName);
}
return indexer;
   }
 

import opennlp.tools.ml.model.DataIndexer; //导入依赖的package包/类
@Override
public MaxentModel doTrain(DataIndexer indexer) throws IOException {

  List<Double> vy = new ArrayList<Double>();
  List<Feature[]> vx = new ArrayList<Feature[]>();

  // outcomes
  int outcomes[] = indexer.getOutcomeList();

  int max_index = 0;
  
  // For each event ...
  for (int i = 0; i < indexer.getContexts().length; i++) {

    int outcome = outcomes[i];
    vy.add(Double.valueOf(outcome));

    int features[] = indexer.getContexts()[i];

    Feature[] x;
    if (bias >= 0) {
      x = new Feature[features.length + 1];
    } else {
      x = new Feature[features.length];
    }

    // for each feature ...
    for (int fi = 0; fi < features.length; fi++) {
      // TODO: SHOUDL BE indexer.getNumTimesEventsSeen()[i] and not fi !!!
      x[fi] = new FeatureNode(features[fi] + 1, indexer.getNumTimesEventsSeen()[i]);
    } 

    if (features.length > 0) {
      max_index = Math.max(max_index, x[features.length - 1].getIndex());
    }
    
    vx.add(x);
  }

  Problem problem = constructProblem(vy, vx, max_index, bias);
  Parameter parameter = new Parameter(solverType, c, eps, p);
  
  Model liblinearModel = Linear.train(problem, parameter);

  Map<String, Integer> predMap = new HashMap<String, Integer>();
  
  String predLabels[] = indexer.getPredLabels();
  for (int i = 0; i < predLabels.length; i++) {
    predMap.put(predLabels[i], i);
  }
  
  return new LiblinearModel(liblinearModel, indexer.getOutcomeLabels(), predMap);
}
 

import opennlp.tools.ml.model.DataIndexer; //导入依赖的package包/类
public MaxentModel doTrain(DataIndexer indexer) throws IOException {
int iterations = getIterations();

MaxentModel model;

int threads = getIntParam("Threads", 1);

model = trainModel(iterations, indexer, true, false, null, 0, threads);

return model;
   }
 

import opennlp.tools.ml.model.DataIndexer; //导入依赖的package包/类
/**
    * Train a model using the GIS algorithm.
    *
    * @param iterations
    *          The number of GIS iterations to perform.
    * @param indexer
    *          The object which will be used for event compilation.
    * @param printMessagesWhileTraining
    *          Determines whether training status messages are written to STDOUT.
    * @param smoothing
    *          Defines whether the created trainer will use smoothing while
    *          training the model.
    * @param modelPrior
    *          The prior distribution for the model.
    * @param cutoff
    *          The number of times a predicate must occur to be used in a model.
    * @return The newly trained model, which can be used immediately or saved to
    *         disk using an opennlp.tools.ml.maxent.io.GISModelWriter object.
    */
   public static GISModel trainModel(int iterations, DataIndexer indexer,
			      boolean printMessagesWhileTraining, boolean smoothing, Prior modelPrior,
			      int cutoff, int threads) {
GISTrainer trainer = new GISTrainer(printMessagesWhileTraining);
trainer.setSmoothing(smoothing);
trainer.setSmoothingObservation(SMOOTHING_OBSERVATION);
if (modelPrior == null) {
    modelPrior = new UniformPrior();
}

return trainer.trainModel(iterations, indexer, modelPrior, cutoff, threads);
   }