Java NameFinderME.find方法代码示例

import opennlp.tools.namefind.NameFinderME; //导入方法依赖的package包/类
public Intent interpret(String query) {
    String[] tokens = WhitespaceTokenizer.INSTANCE.tokenize(query);
    double[] outcome = categorizer.categorize(tokens);
    logger.debug(categorizer.getAllResults(outcome));

    Intent intent = new Intent(categorizer.getBestCategory(outcome));

    for (NameFinderME nameFinderME : nameFinderMEs) {
        Span[] spans = nameFinderME.find(tokens);
        String[] names = Span.spansToStrings(spans, tokens);
        for (int i = 0; i < spans.length; i++) {
            intent.getEntities().put(spans[i].getType(), names[i]);
        }
    }

    logger.debug(intent.toString());

    return intent;
}
 

import opennlp.tools.namefind.NameFinderME; //导入方法依赖的package包/类
@Test
public void testPersonNER()
  throws Exception
{
  URL modelUrl = Thread.currentThread().getContextClassLoader()
      .getResource("models/en-ner-persons.bin");
  assertThat(modelUrl, is(notNullValue()));

  TokenNameFinderModel model = new TokenNameFinderModel(modelUrl);
  assertThat(model, is(notNullValue()));

  NameFinderME nameFinder = new NameFinderME(model);
  String[] tokens = SimpleTokenizer.INSTANCE
      .tokenize("Mr. John Smith of New York, married Anne Green of London today.");
  assertThat(tokens.length, is(15));

  Span[] spans = nameFinder.find(tokens);
  assertThat(spans.length, is(2));

  String[] names = Span.spansToStrings(spans, tokens);
  assertThat(names.length, is(2));
  assertThat(names[0], is("John Smith"));
  assertThat(names[1], is("Anne Green"));
}
 

import opennlp.tools.namefind.NameFinderME; //导入方法依赖的package包/类
@Test
public void testLocationNER()
  throws Exception
{
  URL modelUrl = Thread.currentThread().getContextClassLoader()
      .getResource("models/en-ner-locations.bin");
  assertThat(modelUrl, is(notNullValue()));

  TokenNameFinderModel model = new TokenNameFinderModel(modelUrl);
  assertThat(model, is(notNullValue()));

  NameFinderME nameFinder = new NameFinderME(model);
  String[] tokens = SimpleTokenizer.INSTANCE
      .tokenize("Mr. John Smith of New York, married Anne Green of London today.");
  assertThat(tokens.length, is(15));

  Span[] spans = nameFinder.find(tokens);
  assertThat(spans.length, is(2));

  String[] locations = Span.spansToStrings(spans, tokens);
  assertThat(locations.length, is(2));
  assertThat(locations[0], is("New York"));
  assertThat(locations[1], is("London"));
}
 

import opennlp.tools.namefind.NameFinderME; //导入方法依赖的package包/类
@Test
public void testDateNER()
  throws Exception
{
  URL modelUrl = Thread.currentThread().getContextClassLoader()
      .getResource("models/en-ner-dates.bin");
  assertThat(modelUrl, is(notNullValue()));

  TokenNameFinderModel model = new TokenNameFinderModel(modelUrl);
  assertThat(model, is(notNullValue()));

  NameFinderME nameFinder = new NameFinderME(model);
  String[] tokens = SimpleTokenizer.INSTANCE
      .tokenize("Mr. John Smith of New York, married Anne Green of London today.");
  assertThat(tokens.length, is(15));

  Span[] spans = nameFinder.find(tokens);
  assertThat(spans.length, is(1));

  String[] locations = Span.spansToStrings(spans, tokens);
  assertThat(locations.length, is(1));
  assertThat(locations[0], is("today"));
}
 

import opennlp.tools.namefind.NameFinderME; //导入方法依赖的package包/类
@Test
public void testAddressNER()
  throws Exception
{
  URL modelUrl = Thread.currentThread().getContextClassLoader()
      .getResource("models/en-ner-address.bin");
  assertThat(modelUrl, is(notNullValue()));

  TokenNameFinderModel model = new TokenNameFinderModel(modelUrl);
  assertThat(model, is(notNullValue()));

  NameFinderME nameFinder = new NameFinderME(model);
  String[] tokens = SimpleTokenizer.INSTANCE.tokenize("Send a taxi to 12 Pleasent Street");
  Span[] spans = nameFinder.find(tokens);
  assertThat(spans.length, is(1));

  String[] locations = Span.spansToStrings(spans, tokens);
  assertThat(locations.length, is(1));
  assertThat(locations[0], is("12 Pleasent Street"));
}
 

import opennlp.tools.namefind.NameFinderME; //导入方法依赖的package包/类
public void getAllNameEntitiesfromInput(InputStream stream)
		throws InvalidFormatException, IOException {

	InputStream modelIn = new FileInputStream(nerModelPath);
	TokenNameFinderModel model = new TokenNameFinderModel(modelIn);
	NameFinderME nameFinder = new NameFinderME(model);
	String[] in = IOUtils.toString(stream, "UTF-8").split(" ");

	Span nameE[] = nameFinder.find(in);

	String spanNames = Arrays.toString(Span.spansToStrings(nameE, in));
	spanNames = spanNames.substring(1, spanNames.length() - 1);
	modelIn.close();
	String[] tmp = spanNames.split(",");

	for (String name : tmp) {
		name = name.trim();
		this.locationNameEntities.add(name);
	}
	
	
}
 

import opennlp.tools.namefind.NameFinderME; //导入方法依赖的package包/类
@Nullable public static Map<String,String[]> names(String input) {

        NameFinderME[] finders = (NameFinderME[]) models.get(TokenNameFinderModel.class);
        String[] tokens = tokenizer().tokenize(input);

        Map<String,String[]> x = new HashMap(finders.length);
        for (NameFinderME m : finders) {
            Span[] ss = m.find(tokens);
            if (ss.length > 0)
                x.put(ss[0].getType(), Span.spansToStrings(ss, tokens));
        }

        if (!x.isEmpty()) {
            return x;
        } else {
            return null;
        }
    }
 

import opennlp.tools.namefind.NameFinderME; //导入方法依赖的package包/类
@Override
public List<String> getEntities(EntityType entityCat, String text)
{
    NameFinderME temp = getNameFinderModel(entityCat);
    List<String> entityList = new ArrayList<String>();
    String [] tokens=null;
    tokens = tokenizer.tokenize(text);

    Span nameSpans[] = temp.find(tokens);

    for(Span s: nameSpans)
    {
        StringBuilder sb = new StringBuilder();
        for(int i=s.getStart();i<s.getEnd();i++){
            sb.append(tokens[i]+" ");
        }
        sb.deleteCharAt(sb.length()-1);
        entityList.add(sb.toString());
    }
    return entityList;
}
 

import opennlp.tools.namefind.NameFinderME; //导入方法依赖的package包/类
public Map<String, Set<String>> tokenize(String content) {
    Map<String, Set<String>> namedEntities = Maps.newHashMap();

    List<TextAnnotation> allTextAnnotations = new ArrayList<TextAnnotation>();
    String[] tokens = SimpleTokenizer.INSTANCE.tokenize(content);
    for (Map.Entry<String, TokenNameFinderModel> finderEntry : finders.entrySet()) {
        String type = finderEntry.getKey();
        NameFinderME finder = new NameFinderME(finderEntry.getValue());

        Span[] spans = finder.find(tokens);
        double[] probs = finder.probs(spans);

        for (int ni = 0; ni < spans.length; ni++) {
            allTextAnnotations.add(new TextAnnotation(type, spans[ni], probs[ni]));
        }
    }

    if (allTextAnnotations.size() > 0 ) {
        removeConflicts(allTextAnnotations);
    }
    convertTextAnnotationsToNamedEntities(tokens, allTextAnnotations, namedEntities);

    return namedEntities;
}
 

import opennlp.tools.namefind.NameFinderME; //导入方法依赖的package包/类
private Set<String> nameMatches(String text) {
	Set<String> names = new HashSet<String>();
	NameFinderME nameFinder = new NameFinderME(personModel);
	String tokens[];
	Span matches[];
	for (String sentence : convertToSentences(text)) {
		tokens = convertToTokens(sentence);
		matches = nameFinder.find(tokens);
		List<String> s = Arrays
				.asList(Span.spansToStrings(matches, tokens));
		names.addAll(s);
	}
	nameFinder.clearAdaptiveData();
	return names;
}
 

import opennlp.tools.namefind.NameFinderME; //导入方法依赖的package包/类
@Test
public void testNameFinder(){
	try (InputStream modelIn = BasicActions.class.getClassLoader()
				.getResourceAsStream(Consts.EN_NER_MODEL);){
		
		TokenNameFinderModel model = new TokenNameFinderModel(modelIn);
		NameFinderME nameFinder = new NameFinderME(model);
		Span nameSpans[] = nameFinder.find(testTokenizer());
		System.out.println(Arrays.toString(nameSpans));
		
	} catch (IOException e) {
		e.printStackTrace();
	}
}
 

import opennlp.tools.namefind.NameFinderME; //导入方法依赖的package包/类
public List<Span> recognize(String[] toks) {
	ArrayList<Span> spans = new ArrayList<Span>();
	for (NameFinderME recognizer : recognizers)
		for (Span s : recognizer.find(toks))
			spans.add(s);
	return spans;
}
 

import opennlp.tools.namefind.NameFinderME; //导入方法依赖的package包/类
public Parse[] parsePassageText(String p) throws InvalidFormatException{
	if (!modelsAreInitialized)init();
	//initialize 	 
	SentenceDetectorME sentenceDetector = new SentenceDetectorME(this.sentenceModel);
	NameFinderME nameFinder = new NameFinderME(this.nerModel);
	Parser parser = ParserFactory.create(
			this.parserModel,
			20, // beam size
			0.95); // advance percentage
	//find sentences, tokenize each, parse each, return top parse for each 	 	 
	String[] sentences = sentenceDetector.sentDetect(p);
	Parse[] results = new Parse[sentences.length];
	for (int i=0;i<sentences.length;i++){
		//String[] tks = SimpleTokenizer.INSTANCE.tokenize(sentences[i]);
		
		//StringTokenizer st = new StringTokenizer(tks[i]); 
		//There are several tokenizers available. SimpleTokenizer works best
		Tokenizer tokenizer = SimpleTokenizer.INSTANCE;
		for (int si = 0; si < sentences.length; si++) {
	        Span[] tokenSpans = tokenizer.tokenizePos(sentences[si]);
	        String[] tokens = Span.spansToStrings(tokenSpans, sentences[si]);
	        Span[] names = nameFinder.find(tokens);
	        for (int ni = 0; ni < names.length; ni++) {
	            Span startSpan = tokenSpans[names[ni].getStart()];
	            int nameStart = startSpan.getStart();
	            Span endSpan = tokenSpans[names[ni].getEnd() - 1];
	            int nameEnd = endSpan.getEnd();
	            String name = sentences[si].substring(nameStart, nameEnd);
	            System.out.println(name);
	        }
	    }
		String sent= StringUtils.join(tokenizer," ");
		System.out.println("Found sentence " + sent);
		Parse[] sentResults = ParserTool.parseLine(sent,parser, 1);
		results[i]=sentResults[0];
	}
	return results;
}
 

import opennlp.tools.namefind.NameFinderME; //导入方法依赖的package包/类
@Override
public void execute() throws ExecutionException {
  interrupted = false;
  long startTime = System.currentTimeMillis();
  if(document == null) {
    throw new ExecutionException("No document to process!");
  }
  fireStatusChanged("Running " + this.getName() + " on " + document.getName());
  fireProgressChanged(0);

  AnnotationSet inputAS = document.getAnnotations(inputASName);
  AnnotationSet outputAS = document.getAnnotations(outputASName);
  AnnotationSet sentences = inputAS.get(SENTENCE_ANNOTATION_TYPE);

  int nbrDone = 0;
  int nbrSentences = sentences.size();

  for (Annotation sentence : sentences) {
    /* For each input Sentence annotation, produce a list of
     * Token.string values and the data structure for translating
     * offsets.		   */
    AnnotationSet tokenSet = Utils.getContainedAnnotations(inputAS, sentence, TOKEN_ANNOTATION_TYPE);
    Sentence tokens = new Sentence(tokenSet, TOKEN_STRING_FEATURE_NAME, TOKEN_CATEGORY_FEATURE_NAME);
    String[] strings = tokens.getStrings();

    // Run each NameFinder over the sentence
    for (NameFinderME finder : finders.keySet()) {
      String type = finders.get(finder);
      Span[] spans = finder.find(strings);

      for (Span span : spans) {
        // Translate the offsets and create the output NE annotation
        long start = tokens.getStartOffset(span);
        long end = tokens.getEndOffset(span);
        FeatureMap fm = Factory.newFeatureMap();
        fm.put("source", "OpenNLP");
        try {
          outputAS.add(start, end, type, fm);
        }
        catch (InvalidOffsetException e) {
          throw new ExecutionException(e);
        }

        if(isInterrupted()) { 
          throw new ExecutionInterruptedException("Execution of " + 
              this.getName() + " has been abruptly interrupted!");
        }
      } // end loop over names from 1 finder in 1 sentence
    } // end loop over NameFinders within one sentence
    nbrDone++;
    fireProgressChanged((int)(100 * nbrDone / nbrSentences));
  } // end for sentence : sentences

  fireProcessFinished();
  fireStatusChanged("Finished " + this.getName() + " on " + document.getName()
      + " in " + NumberFormat.getInstance().format(
          (double)(System.currentTimeMillis() - startTime) / 1000)
          + " seconds!");
}
 

import opennlp.tools.namefind.NameFinderME; //导入方法依赖的package包/类
@Override
public List<Entity> retrieve(final String input) {
  LOG.info("retrieve ...");

  final List<Entity> list = new ArrayList<>();
  final String[] sentences = FoxTextUtil.getSentences(input);
  LOG.debug("sentences: " + sentences.length);

  for (int i = 0; i < tokenNameFinderModels.length; i++) {
    if (tokenNameFinderModels[i] != null) {
      final NameFinderME nameFinder = new NameFinderME(tokenNameFinderModels[i]);
      for (final String sentence : sentences) {
        final String[] tokens = FoxTextUtil.getSentenceToken(sentence);
        LOG.debug("tokens: " + tokens.length);
        if ((tokens.length > 0) && tokens[tokens.length - 1].trim().isEmpty()) {
          tokens[tokens.length - 1] = ".";
        }

        final Span[] nameSpans = nameFinder.find(tokens);
        nameFinder.probs(nameSpans);
        for (int ii = 0; ii < nameSpans.length; ii++) {
          final Span span = nameSpans[ii];

          String word = "";
          for (int j = 0; j < (span.getEnd() - span.getStart()); j++) {
            word += tokens[span.getStart() + j] + " ";
          }
          word = word.trim();

          final float p = Entity.DEFAULT_RELEVANCE;
          // if ((FoxCfg.get("openNLPDefaultRelevance") != null)
          // && !Boolean.valueOf(FoxCfg.get("openNLPDefaultRelevance"))) {
          // p = Double.valueOf(probs[ii]).floatValue();
          // }
          final String cl = mapTypeToSupportedType(span.getType());
          if (cl != EntityClassMap.getNullCategory()) {
            list.add(getEntity(word, cl, p, getToolName()));
          }
        }
      }
      nameFinder.clearAdaptiveData();
    }
  }
  // TRACE
  if (LOG.isTraceEnabled()) {
    LOG.trace(list);
  } // TRACE
  return list;
}