Java NameFinderME类代码示例

import opennlp.tools.namefind.NameFinderME; //导入依赖的package包/类
public Intent interpret(String query) {
    String[] tokens = WhitespaceTokenizer.INSTANCE.tokenize(query);
    double[] outcome = categorizer.categorize(tokens);
    logger.debug(categorizer.getAllResults(outcome));

    Intent intent = new Intent(categorizer.getBestCategory(outcome));

    for (NameFinderME nameFinderME : nameFinderMEs) {
        Span[] spans = nameFinderME.find(tokens);
        String[] names = Span.spansToStrings(spans, tokens);
        for (int i = 0; i < spans.length; i++) {
            intent.getEntities().put(spans[i].getType(), names[i]);
        }
    }

    logger.debug(intent.toString());

    return intent;
}
 

import opennlp.tools.namefind.NameFinderME; //导入依赖的package包/类
protected void loadNameFinders(String language, String modelDirectory) throws IOException {
  //<start id="maxent.examples.namefinder.setup"/> 
  File modelFile;

  File[] models //<co id="nfe.findmodels"/>
    = findNameFinderModels(language, modelDirectory);
  modelNames = new String[models.length];
  finders = new NameFinderME[models.length];

  for (int fi = 0; fi < models.length; fi++) {
    modelFile = models[fi];
    modelNames[fi] = modelNameFromFile(language, modelFile); //<co id="nfe.modelname"/>
    
    log.info("Loading model {}", modelFile); 
    InputStream modelStream = new FileInputStream(modelFile);
    TokenNameFinderModel model = //<co id="nfe.modelreader"/>
        new TokenNameFinderModel(modelStream);
    finders[fi] = new NameFinderME(model);
    
  }

}
 

import opennlp.tools.namefind.NameFinderME; //导入依赖的package包/类
/**
 * Private constructor to allow only one {@link OpenNLP} for each Thread
 *
 * @throws IllegalStateException if an error occurred from {@link LoaderNLP} or {@link PropertiesManager}
 */
private OpenNLP() {
    try {
        detector = new SentenceDetectorME(LoaderNLP.getSentenceModel());
        tokenizer = new TokenizerME(LoaderNLP.getTokenizerModel());
        tagger = new POSTaggerME(LoaderNLP.getPosModel());
        nameFinderOrg = new NameFinderME(LoaderNLP.getTokenNameFinderModelOrg());
        nameFinderLoc = new NameFinderME(LoaderNLP.getTokenNameFinderModelLoc());
        nameFinderPers = new NameFinderME(LoaderNLP.getTokenNameFinderModelPers());
        InputStream inputStream = new FileInputStream(PROPERTIES_MANAGER.getProperty("nlp.dictionaries.path"));
        lemmatizer = new SimpleLemmatizer(inputStream);
        inputStream.close();
    } catch (IllegalArgumentException | IOException e) {
        LOGGER.error(e.getMessage());
        throw new IllegalStateException(e);
    }
}
 

import opennlp.tools.namefind.NameFinderME; //导入依赖的package包/类
@Override
public void buildModel(String entityType) {
  try {
    System.out.println("\tBuilding Model using " + annotatedSentences.size() + " annotations");
    System.out.println("\t\treading training data...");
    Charset charset = Charset.forName("UTF-8");
    ObjectStream<String> lineStream =
            new PlainTextByLineStream(new MarkableFileInputStreamFactory(params.getAnnotatedTrainingDataFile()), charset);
    ObjectStream<NameSample> sampleStream = new NameSampleDataStream(lineStream);

    TokenNameFinderModel model;
    model = NameFinderME.train("en", entityType, sampleStream, null);
    sampleStream.close();
    OutputStream modelOut = new BufferedOutputStream(new FileOutputStream(params.getModelFile()));
    model.serialize(modelOut);
    if (modelOut != null) {
      modelOut.close();
    }
    System.out.println("\tmodel generated");
  } catch (Exception e) {
  }
}
 

import opennlp.tools.namefind.NameFinderME; //导入依赖的package包/类
public Set<String> find(String content, String field) {
    try {
        if (!nameFinderModels.containsKey(field)) {
            throw new ElasticsearchException("Could not find field [{}], possible values {}", field, nameFinderModels.keySet());
        }
        TokenNameFinderModel finderModel= nameFinderModels.get(field);
        if (threadLocal.get() == null || !threadLocal.get().equals(finderModel)) {
            threadLocal.set(finderModel);
        }

        String[] tokens = SimpleTokenizer.INSTANCE.tokenize(content);
        Span spans[] = new NameFinderME(finderModel).find(tokens);
        String[] names = Span.spansToStrings(spans, tokens);
        return Sets.newHashSet(names);
    } finally {
        threadLocal.remove();
    }
}
 

import opennlp.tools.namefind.NameFinderME; //导入依赖的package包/类
@Test
public void testPersonNER()
  throws Exception
{
  URL modelUrl = Thread.currentThread().getContextClassLoader()
      .getResource("models/en-ner-persons.bin");
  assertThat(modelUrl, is(notNullValue()));

  TokenNameFinderModel model = new TokenNameFinderModel(modelUrl);
  assertThat(model, is(notNullValue()));

  NameFinderME nameFinder = new NameFinderME(model);
  String[] tokens = SimpleTokenizer.INSTANCE
      .tokenize("Mr. John Smith of New York, married Anne Green of London today.");
  assertThat(tokens.length, is(15));

  Span[] spans = nameFinder.find(tokens);
  assertThat(spans.length, is(2));

  String[] names = Span.spansToStrings(spans, tokens);
  assertThat(names.length, is(2));
  assertThat(names[0], is("John Smith"));
  assertThat(names[1], is("Anne Green"));
}
 

import opennlp.tools.namefind.NameFinderME; //导入依赖的package包/类
@Test
public void testLocationNER()
  throws Exception
{
  URL modelUrl = Thread.currentThread().getContextClassLoader()
      .getResource("models/en-ner-locations.bin");
  assertThat(modelUrl, is(notNullValue()));

  TokenNameFinderModel model = new TokenNameFinderModel(modelUrl);
  assertThat(model, is(notNullValue()));

  NameFinderME nameFinder = new NameFinderME(model);
  String[] tokens = SimpleTokenizer.INSTANCE
      .tokenize("Mr. John Smith of New York, married Anne Green of London today.");
  assertThat(tokens.length, is(15));

  Span[] spans = nameFinder.find(tokens);
  assertThat(spans.length, is(2));

  String[] locations = Span.spansToStrings(spans, tokens);
  assertThat(locations.length, is(2));
  assertThat(locations[0], is("New York"));
  assertThat(locations[1], is("London"));
}
 

import opennlp.tools.namefind.NameFinderME; //导入依赖的package包/类
@Test
public void testDateNER()
  throws Exception
{
  URL modelUrl = Thread.currentThread().getContextClassLoader()
      .getResource("models/en-ner-dates.bin");
  assertThat(modelUrl, is(notNullValue()));

  TokenNameFinderModel model = new TokenNameFinderModel(modelUrl);
  assertThat(model, is(notNullValue()));

  NameFinderME nameFinder = new NameFinderME(model);
  String[] tokens = SimpleTokenizer.INSTANCE
      .tokenize("Mr. John Smith of New York, married Anne Green of London today.");
  assertThat(tokens.length, is(15));

  Span[] spans = nameFinder.find(tokens);
  assertThat(spans.length, is(1));

  String[] locations = Span.spansToStrings(spans, tokens);
  assertThat(locations.length, is(1));
  assertThat(locations[0], is("today"));
}
 

import opennlp.tools.namefind.NameFinderME; //导入依赖的package包/类
@Test
public void testAddressNER()
  throws Exception
{
  URL modelUrl = Thread.currentThread().getContextClassLoader()
      .getResource("models/en-ner-address.bin");
  assertThat(modelUrl, is(notNullValue()));

  TokenNameFinderModel model = new TokenNameFinderModel(modelUrl);
  assertThat(model, is(notNullValue()));

  NameFinderME nameFinder = new NameFinderME(model);
  String[] tokens = SimpleTokenizer.INSTANCE.tokenize("Send a taxi to 12 Pleasent Street");
  Span[] spans = nameFinder.find(tokens);
  assertThat(spans.length, is(1));

  String[] locations = Span.spansToStrings(spans, tokens);
  assertThat(locations.length, is(1));
  assertThat(locations[0], is("12 Pleasent Street"));
}
 

import opennlp.tools.namefind.NameFinderME; //导入依赖的package包/类
public static void trainNameFinder(final String inResource, String outFile) throws IOException {
    InputStreamFactory inputStreamFactory = new InputStreamFactory() {
        @Override
        public InputStream createInputStream() throws IOException {
            return Trainer.class.getResourceAsStream(inResource);
        }
    };
    InputStream in = Trainer.class.getResourceAsStream(inResource);
    NameSampleDataStream samples = new NameSampleDataStream(new PlainTextByLineStream(inputStreamFactory, StandardCharsets.UTF_8));
    TrainingParameters trainingParameters = new TrainingParameters();
    trainingParameters.put(TrainingParameters.ITERATIONS_PARAM, "5");
    trainingParameters.put(TrainingParameters.CUTOFF_PARAM, "200");
    byte[] featureGeneratorBytes = null;
    Map<String, Object> resources = Collections.<String, Object>emptyMap();
    SequenceCodec<String> seqCodec = new BioCodec();
    TokenNameFinderFactory tokenNameFinderFactory = TokenNameFinderFactory.create(null, featureGeneratorBytes, resources, seqCodec);
    TokenNameFinderModel model = NameFinderME.train("en", "person", samples, trainingParameters, tokenNameFinderFactory);
    //NameFinderME.train("en", "person", samples, Collections.<String, Object>emptyMap(), 200, 5);
    samples.close();
    FileOutputStream out = new FileOutputStream(outFile);
    model.serialize(out);
    out.close();
}
 

import opennlp.tools.namefind.NameFinderME; //导入依赖的package包/类
public void getAllNameEntitiesfromInput(InputStream stream)
		throws InvalidFormatException, IOException {

	InputStream modelIn = new FileInputStream(nerModelPath);
	TokenNameFinderModel model = new TokenNameFinderModel(modelIn);
	NameFinderME nameFinder = new NameFinderME(model);
	String[] in = IOUtils.toString(stream, "UTF-8").split(" ");

	Span nameE[] = nameFinder.find(in);

	String spanNames = Arrays.toString(Span.spansToStrings(nameE, in));
	spanNames = spanNames.substring(1, spanNames.length() - 1);
	modelIn.close();
	String[] tmp = spanNames.split(",");

	for (String name : tmp) {
		name = name.trim();
		this.locationNameEntities.add(name);
	}
	
	
}
 

import opennlp.tools.namefind.NameFinderME; //导入依赖的package包/类
@Nullable public static Map<String,String[]> names(String input) {

        NameFinderME[] finders = (NameFinderME[]) models.get(TokenNameFinderModel.class);
        String[] tokens = tokenizer().tokenize(input);

        Map<String,String[]> x = new HashMap(finders.length);
        for (NameFinderME m : finders) {
            Span[] ss = m.find(tokens);
            if (ss.length > 0)
                x.put(ss[0].getType(), Span.spansToStrings(ss, tokens));
        }

        if (!x.isEmpty()) {
            return x;
        } else {
            return null;
        }
    }
 

import opennlp.tools.namefind.NameFinderME; //导入依赖的package包/类
@Override
public List<String> getEntities(EntityType entityCat, String text)
{
    NameFinderME temp = getNameFinderModel(entityCat);
    List<String> entityList = new ArrayList<String>();
    String [] tokens=null;
    tokens = tokenizer.tokenize(text);

    Span nameSpans[] = temp.find(tokens);

    for(Span s: nameSpans)
    {
        StringBuilder sb = new StringBuilder();
        for(int i=s.getStart();i<s.getEnd();i++){
            sb.append(tokens[i]+" ");
        }
        sb.deleteCharAt(sb.length()-1);
        entityList.add(sb.toString());
    }
    return entityList;
}
 

import opennlp.tools.namefind.NameFinderME; //导入依赖的package包/类
@Override
public void startStage(StageConfiguration config) {

  // parse the config to map the params properly
  textField = config.getProperty("textField", textField);
  peopleField = config.getProperty("peopleField", peopleField);
  posTextField = config.getProperty("posTextField", posTextField);

  try {
    // Sentence finder
    SentenceModel sentModel = new SentenceModel(new FileInputStream(sentenceModelFile));
    sentenceDetector = new SentenceDetectorME(sentModel);
    // tokenizer
    TokenizerModel tokenModel = new TokenizerModel(new FileInputStream(tokenModelFile));
    tokenizer = new TokenizerME(tokenModel);
    // person name finder
    TokenNameFinderModel nameModel = new TokenNameFinderModel(new FileInputStream(personModelFile));
    nameFinder = new NameFinderME(nameModel);
    // load the part of speech tagger.
    posTagger = new POSTaggerME(new POSModel(new FileInputStream(posModelFile)));
  } catch (IOException e) {
    log.info("Error loading up OpenNLP Models. {}", e.getLocalizedMessage());
    e.printStackTrace();
  }
}
 

import opennlp.tools.namefind.NameFinderME; //导入依赖的package包/类
public Map<String, Set<String>> tokenize(String content) {
    Map<String, Set<String>> namedEntities = Maps.newHashMap();

    List<TextAnnotation> allTextAnnotations = new ArrayList<TextAnnotation>();
    String[] tokens = SimpleTokenizer.INSTANCE.tokenize(content);
    for (Map.Entry<String, TokenNameFinderModel> finderEntry : finders.entrySet()) {
        String type = finderEntry.getKey();
        NameFinderME finder = new NameFinderME(finderEntry.getValue());

        Span[] spans = finder.find(tokens);
        double[] probs = finder.probs(spans);

        for (int ni = 0; ni < spans.length; ni++) {
            allTextAnnotations.add(new TextAnnotation(type, spans[ni], probs[ni]));
        }
    }

    if (allTextAnnotations.size() > 0 ) {
        removeConflicts(allTextAnnotations);
    }
    convertTextAnnotationsToNamedEntities(tokens, allTextAnnotations, namedEntities);

    return namedEntities;
}