Java NamedEntityTagAnnotation类代码示例

import edu.stanford.nlp.ling.CoreAnnotations.NamedEntityTagAnnotation; //导入依赖的package包/类
/**
 * Checks for the presence of some critical annotations. In the case some of those entered among the
 * parameters is missing, the texts needs to be re-annotated.
 * 
 * @param a
 *          annotation
 * @param r
 *          reader with the desired annotations
 * @return
 */
private boolean needsReannotation(Annotation a, KpeReader r) {
  List<CoreMap> sentences = a.get(SentencesAnnotation.class);
  List<CoreLabel> tokens = a.get(TokensAnnotation.class);
  if (tokens == null || sentences == null || tokens.size() == 0 || sentences.size() == 0) {
    return true;
  }
  Set<Class<?>> sentenceAnnotations = sentences.get(0).keySet();
  Set<Class<?>> tokenAnnotations = tokens.get(0).keySet();
  if ((r.getIsMweOn() && !tokenAnnotations.contains(MWEAnnotation.class))
      || (r.getIsNeOn() && !tokenAnnotations.contains(NamedEntityTagAnnotation.class))) {
    return true;
  }
  if (r.getIsSyntaxOn() && !sentenceAnnotations.contains(TreeAnnotation.class)) {
    return true;
  }
  return false;
}
 

import edu.stanford.nlp.ling.CoreAnnotations.NamedEntityTagAnnotation; //导入依赖的package包/类
public static void en(final String[] a) {

    /*
     * for (Entity e : new NERStanford().retrieve(FoxConst.EXAMPLE_1)) NERStanford.LOG.info(e);
     */

    final Properties props = new Properties();
    props.setProperty("annotators", "tokenize, ssplit, pos, lemma, ner, parse, dcoref, relation");
    final StanfordCoreNLP pipeline = new StanfordCoreNLP(props);
    final Annotation ann =
        new Annotation("Stanford University is located in California. It is a great university.");
    pipeline.annotate(ann);

    for (final CoreMap sentence : ann.get(SentencesAnnotation.class)) {
      for (final CoreLabel token : sentence.get(TokensAnnotation.class)) {
        System.out.println(token.get(NamedEntityTagAnnotation.class));
        System.out.println(token.get(CoreAnnotations.AnswerAnnotation.class));
        /*
         * Tree tree = sentence.get(TreeAnnotation.class); System.out.println(tree);
         * System.out.println(tree.score());
         */
      }
    }

  }
 

import edu.stanford.nlp.ling.CoreAnnotations.NamedEntityTagAnnotation; //导入依赖的package包/类
/**
 *
 */
public PreNERCoreLabelWrapper(final CoreLabel cl) {
  this.orig = new TokenizedCoreLabelWrapper(cl);

  this.posTag = Optional.ofNullable(cl.get(PartOfSpeechAnnotation.class));
  this.nerTag = Optional.ofNullable(cl.get(NamedEntityTagAnnotation.class));
  this.lemmaTag = Optional.ofNullable(cl.get(LemmaAnnotation.class));
}
 

import edu.stanford.nlp.ling.CoreAnnotations.NamedEntityTagAnnotation; //导入依赖的package包/类
public Multimap<String, String> annotateText(String text) {
	loadResource();
	Multimap<String, String> taggerTokens = ArrayListMultimap.create();
	Annotation document = new Annotation(text);
	pipeline.annotate(document);

	for (CoreLabel token : document.get(TokensAnnotation.class)) {
		String ne = token.get(NamedEntityTagAnnotation.class);
		String word = token.get(TextAnnotation.class);
		taggerTokens.put(ne, word);
	}
	return taggerTokens;
}
 

import edu.stanford.nlp.ling.CoreAnnotations.NamedEntityTagAnnotation; //导入依赖的package包/类
public List<String> tagTokens(String text) {

		List<String> tagged = new ArrayList<String>();

		Annotation document = runPipeline(text);

		// these are all the sentences in this document
		// a CoreMap is essentially a Map that uses class objects as keys
		// and has values with custom types
		List<CoreMap> sentences = document.get(SentencesAnnotation.class);

		for (CoreMap sentence : sentences) {
			// traversing the words in the current sentence
			// a CoreLabel is a CoreMap with additional token-specific methods
			for (CoreLabel token : sentence.get(TokensAnnotation.class)) {
				// this is the text of the token
				String word = token.get(TextAnnotation.class);
				// this is the POS tag of the token
				String pos = token.get(PartOfSpeechAnnotation.class);
				// this is the NER label of the token
				String ne = token.get(NamedEntityTagAnnotation.class);
				// this is the lemma of the token
				String lemma = token.get(LemmaAnnotation.class);
				// this is the sentence index
				int sentId = token.get(SentenceIndexAnnotation.class);

				tagged.add(word + "/" + pos + "/" + ne + "/" + lemma + "/" + sentId);
			}

		}

		return tagged;
	}
 

import edu.stanford.nlp.ling.CoreAnnotations.NamedEntityTagAnnotation; //导入依赖的package包/类
public static List<Pair<Integer, Integer>> getNamedEntityMentions(
		CoreMap sentence) {
	List<Pair<Integer, Integer>> offsets = new ArrayList<Pair<Integer, Integer>>();
	String prevTag = "O";
	int tid = 0;
	int start = -1;
	for (CoreLabel token : sentence.get(TokensAnnotation.class)) {
		String tag = token.get(NamedEntityTagAnnotation.class);
		if (!validTags.contains(tag)) {
			tag = "O";
		}
		if (tag.equals(prevTag)) {

		} else {
			if (tag.equals("O")) {
				offsets.add(Pair.makePair(start, tid));
				start = -1;
			} else {
				if (prevTag.equals("O")) {
					start = tid;
				} else {
					offsets.add(Pair.makePair(start, tid));
					start = tid;
				}
			}
		}
		prevTag = tag;
		tid++;
	}
	if (!prevTag.equals("O")) {
		offsets.add(Pair.makePair(start, tid));
	}
	return offsets;
}
 

import edu.stanford.nlp.ling.CoreAnnotations.NamedEntityTagAnnotation; //导入依赖的package包/类
public static List<Pair<Integer, Integer>> getAllNamedEntityMentions(CoreMap sentence) {
	List<Pair<Integer, Integer>> offsets = new ArrayList<Pair<Integer, Integer>>();
	String prevTag = "O";
	int tid = 0;
	int start = -1;
	for (CoreLabel token : sentence.get(TokensAnnotation.class)) {
		String tag = token.get(NamedEntityTagAnnotation.class);
		// if (!validTags.contains(tag)) {
		// tag = "O";
		// }
		if (tag.equals(prevTag)) {

		} else {
			if (tag.equals("O")) {
				offsets.add(Pair.makePair(start, tid));
				start = -1;
			} else {
				if (prevTag.equals("O")) {
					start = tid;
				} else {
					offsets.add(Pair.makePair(start, tid));
					start = tid;
				}
			}
		}
		prevTag = tag;
		tid++;
	}
	if (!prevTag.equals("O")) {
		offsets.add(Pair.makePair(start, tid));
	}
	return offsets;
}
 

import edu.stanford.nlp.ling.CoreAnnotations.NamedEntityTagAnnotation; //导入依赖的package包/类
public static List<Pair<Integer, Integer>> getNamedEntityMentions(CoreMap sentence) {
	List<Pair<Integer, Integer>> offsets = new ArrayList<Pair<Integer, Integer>>();
	String prevTag = "O";
	int tid = 0;
	int start = -1;
	for (CoreLabel token : sentence.get(TokensAnnotation.class)) {
		String tag = token.get(NamedEntityTagAnnotation.class);
		if (!validTags.contains(tag)) {
			tag = "O";
		}
		if (tag.equals(prevTag)) {

		} else {
			if (tag.equals("O")) {
				offsets.add(Pair.makePair(start, tid));
				start = -1;
			} else {
				if (prevTag.equals("O")) {
					start = tid;
				} else {
					offsets.add(Pair.makePair(start, tid));
					start = tid;
				}
			}
		}
		prevTag = tag;
		tid++;
	}
	if (!prevTag.equals("O")) {
		offsets.add(Pair.makePair(start, tid));
	}
	return offsets;
}
 

import edu.stanford.nlp.ling.CoreAnnotations.NamedEntityTagAnnotation; //导入依赖的package包/类
public static List<String> recognizeNE(String tweet) throws IOException {
	// Find the NE in tweets using Stanford CRF tagger.
	tweet = tweet.replaceAll("@", "").replaceAll("#", "");

	// create an empty Annotation just with the given text
	final Annotation doc = new Annotation(tweet);
	pipeline.annotate(doc);

	// these are all the sentences in this document
	// a CoreMap is essentially a Map that uses class objects as keys and
	// has values with custom types
	final List<CoreMap> sentences = doc.get(SentencesAnnotation.class);
	final List<String> neList=new ArrayList<String>();

	for (CoreMap sentence : sentences) {
		// traversing the words in the current sentence
		// a CoreLabel is a CoreMap with additional token-specific methods
		for (CoreLabel token : sentence.get(TokensAnnotation.class)) {
			// this is the text of the token
			String word = token.get(TextAnnotation.class);
			// this is the NER label of the token
			String ne = token.get(NamedEntityTagAnnotation.class);
			if(ne.equals("PERSON") || ne.equals("ORGANIZATION") || ne.equals("LOCATION"))
				neList.add(word);
		}
	}

	return neList;
}
 

import edu.stanford.nlp.ling.CoreAnnotations.NamedEntityTagAnnotation; //导入依赖的package包/类
public void DoAll(String data, String[] TokenizedData, String[] POSTags, String[] LEMMA, Map<String,String> parentEdge,
    Map<String,ArrayList<String>> childrenEdge)
{
  //if(document == null)
  {
    document = new Annotation(data);
      pipelineTags.annotate(document);
  }
    int i=0;
  List<CoreLabel> tokens = document.get(TokensAnnotation.class);

  for (CoreLabel token : tokens) {
    
          String wPOS = token.get(PartOfSpeechAnnotation.class);
          String wNER = token.get(NamedEntityTagAnnotation.class);
          String wLEMMA = token.get(LemmaAnnotation.class)  ;
          TokenizedData[i]= token.toString();
          POSTags[i]= wPOS;
          LEMMA[i]= wLEMMA;
      i++;
  }
  List<CoreMap> sentences = document.get(SentencesAnnotation.class);
  if(sentences.size()>0){
  SemanticGraph tree = sentences.get(0).get(BasicDependenciesAnnotation.class);
  //System.out.println(data+"\n"+tree.toString()+"\n");
  createEdgeMap(tree,parentEdge,childrenEdge);
  }
      
}
 

import edu.stanford.nlp.ling.CoreAnnotations.NamedEntityTagAnnotation; //导入依赖的package包/类
public String[] NERTagger(String data)
{
  String[] NERTags = null;
  int i=0;
  List<CoreLabel> tokens = Tokenizer( data);
  for (CoreLabel token : tokens) {
          String word = token.get(NamedEntityTagAnnotation.class);
          NERTags[i]= word;
          i++;
  }
         
  return NERTags;
      
}
 

import edu.stanford.nlp.ling.CoreAnnotations.NamedEntityTagAnnotation; //导入依赖的package包/类
@Override
public void annotate(Annotation document) {
	if(document.has(TokensAnnotation.class)){
		String[] ner = new String[3];
		List<CoreLabel> tokens = document.get(TokensAnnotation.class);
		CoreLabel prev = null;
		for(CoreLabel token : tokens){
			ner[0] = ner[1];
			ner[1] = ner[2];
			ner[2] = token.get(NamedEntityTagAnnotation.class);				
			
			if(ner[1] != null && !ner[1].equals("O") && ner[2] != null && !ner[2].equals("O")){
				// Two named entities in a row
			}else if(ner[0] != null && !ner[0].equals("O") && ner[2] != null && !ner[2].equals("O") && prev.get(TextAnnotation.class).equals(",")){
				//Named entity comma named entity
				String textRep = ner[0] + "," + ner[2];
				//System.out.println(textRep);
				if(nerPairs.containsKey(textRep)){
					nerPairs.put(textRep, nerPairs.get(textRep)+1);
				}else{
					nerPairs.put(textRep, 1);
				}
				//System.out.println(nerPairs);
				if (ner[0].equals("LOCATION") && ner[2].equals("LOCATION")) {
					prev.set(TextAnnotation.class, "and");
					prev.set(ValueAnnotation.class, "and");
					prev.set(PartOfSpeechAnnotation.class, "CC");
					prev.set(LemmaAnnotation.class, "and");
				}
			}
			
			prev = token;
		}
	}
}
 

import edu.stanford.nlp.ling.CoreAnnotations.NamedEntityTagAnnotation; //导入依赖的package包/类
public static void main(String args[])
{
	Properties props = new Properties();
	props.setProperty("annotators", "tokenize, cleanxml, ssplit,pos,lemma,ner");
	
	StanfordCoreNLP pipeline = new StanfordCoreNLP(props);
	pipeline.addAnnotator(new TimeAnnotator("sutime", props));
	String text = "<mydata> refeer</mydata>today is 12 jan 2016. what is tommorow? Who is Avtar? Does he work at Apple or Google? Sumit was born on 13 feb,2011.";

	Annotation document = new Annotation(text);
	pipeline.annotate(document);
    System.out.println(document.get(CoreAnnotations.TextAnnotation.class));
    List<CoreMap> timexAnnsAll = document.get(TimeAnnotations.TimexAnnotations.class);
    for (CoreMap cm : timexAnnsAll) {
    List<CoreLabel> tokens = cm.get(CoreAnnotations.TokensAnnotation.class);
    TimeData td=new TimeData();
    td.setTime(cm.get(TimeExpression.Annotation.class).getTemporal().toISOString());
    td.setStart(tokens.get(0).get(CoreAnnotations.CharacterOffsetBeginAnnotation.class));
    td.setEnd(tokens.get(tokens.size() - 1).get(CoreAnnotations.CharacterOffsetEndAnnotation.class));
    }
 
	List<CoreMap> sentences = document.get(SentencesAnnotation.class);
	for(CoreMap sentence: sentences) {
	  // traversing the words in the current sentence
	  // a CoreLabel is a CoreMap with additional token-specific methods
		System.out.println("in sent");
	  for (CoreLabel token: sentence.get(TokensAnnotation.class)) {
	    // this is the text of the token
		  System.out.println("in token");
	    String word = token.get(TextAnnotation.class);
	    // this is the POS tag of the token
	    String pos = token.get(PartOfSpeechAnnotation.class);
	    // this is the NER label of the token
	    String ne = token.get(NamedEntityTagAnnotation.class);
	    System.out.println("word : "+word+" pos: "+pos+" ner: "+ne);
	    
	  }

	}

}
 

import edu.stanford.nlp.ling.CoreAnnotations.NamedEntityTagAnnotation; //导入依赖的package包/类
public Iterator findTokens() throws IOException
{
	/*char[] c = new char[256];
    int sz = 0;
    StringBuilder b = new StringBuilder();
    
    while ((sz = input.read(c)) >= 0) {
      b.append(c, 0, sz);
    }*/
    //String text = b.toString();
	if (!input.incrementToken()) return null;
    String text;
    text = input.getAttribute(CharTermAttribute.class).toString();
	// read some text in the text variable
	//System.out.println("before annotation");
	Annotation document = new Annotation(text);
	// these are all the sentences in this document
	// a CoreMap is essentially a Map that uses class objects as keys and has values with custom types
	pipeline.annotate(document);
	List<CoreMap> timexAnnsAll = document.get(TimeAnnotations.TimexAnnotations.class);
    for (CoreMap cm : timexAnnsAll) {
    List<CoreLabel> tokens = cm.get(CoreAnnotations.TokensAnnotation.class);
    TimeData td=new TimeData();
    td.setTime(cm.get(TimeExpression.Annotation.class).getTemporal().toString());
    td.setStart(tokens.get(0).get(CoreAnnotations.CharacterOffsetBeginAnnotation.class));
    td.setEnd(tokens.get(tokens.size() - 1).get(CoreAnnotations.CharacterOffsetEndAnnotation.class));
    timeQueue.add(td);
    }
	List<CoreMap> sentences = document.get(SentencesAnnotation.class);
	
	//System.out.println("after annotation and sentence getting"+sentences.size());
	for(CoreMap sentence: sentences) {
	  // traversing the words in the current sentence
	  // a CoreLabel is a CoreMap with additional token-specific methods
	  for (CoreLabel token: sentence.get(TokensAnnotation.class)) {
	    // this is the text of the token
		  System.out.println("in token");
	    String word = token.get(TextAnnotation.class);
	    // this is the POS tag of the token
	    String pos = token.get(PartOfSpeechAnnotation.class);
	    // this is the NER label of the token
	    String ne = token.get(NamedEntityTagAnnotation.class);
	   // System.out.println("word : "+word+" pos: "+pos+" ner: "+ne);
	    TokenData tok=new TokenData();
	    tok.setNER(ne);
	    tok.setToken(word);
	    tok.setPOS(pos);
	    tokenQueue.add(tok);
	  }

	}
	Iterator<TokenData> it=tokenQueue.iterator();
	itr_cpy=tokenQueue.iterator();
	tokenOffset=0;
	start=0;
	end=0;
	return it;
}
 

import edu.stanford.nlp.ling.CoreAnnotations.NamedEntityTagAnnotation; //导入依赖的package包/类
@Override
	public List<ExtractNamedEntity> ExtractNamedEntityFile(File filePath) throws Exception {
		
		List<String> lstData=ReadRawData(filePath);
        List<ExtractNamedEntity> lstEntityList=new ArrayList<>();
        Properties props = new Properties();
        boolean useRegexner = true;
       // java.net.URL url = getClass().getClassLoader().getResource("data.txt");
		//File file = new File(url.toURI());
        if (useRegexner) 
        {
          props.put("annotators", "tokenize, ssplit, pos, lemma, ner,regexner");
          props.put("regexner.mapping", "data.txt");
          
        } else 
        {
          props.put("annotators", "tokenize, ssplit, pos,lemma, ner");
        }
        int wordPosition=0;
        StanfordCoreNLP pipeline = new StanfordCoreNLP(props);
        for(String str:lstData){
        	Annotation document = new Annotation(str);
	          pipeline.annotate(document);
	          
	          List<CoreMap> sentences = document.get(SentencesAnnotation.class);
	          
	           for (CoreMap sentence : sentences) {
	               for (CoreLabel token : sentence.get(TokensAnnotation.class)) 
	               {
	                   String tToken = token.get(NamedEntityTagAnnotation.class);
	                   String word = token.get(TextAnnotation.class);
	                   //lstWordsLocation.add(new ExtractNamedEntity(word,wordPosition));
	                   wordPosition++;
	                   if(true)
	                   {
	                      if(tToken.equalsIgnoreCase("O"))
	                      {
	                           continue;
	                      }
	                      lstEntityList.add(new ExtractNamedEntity(word,tToken,wordPosition));
	                   }
/*	                   else
	                   {
	                      lstEntityList.add(new ExtractNamedEntity(word,tToken,wordPosition));
	                   }*/
	               }
	              // lstEntityList.add(new ExtractNamedEntity(".",".",0));
	           }
	           
        }
        
        List<ExtractNamedEntity> lstFinal=new ArrayList<>();
        for(int i=0;i<lstEntityList.size();i++)
        {
            try
            {
            String wordEnt=lstEntityList.get(i).word+" ";
            String entityEnt=lstEntityList.get(i).ner;
            int positionEnt=lstEntityList.get(i).position;
            int positionNextEnt=lstEntityList.get(i+1).position;
            while ((positionEnt+1)==positionNextEnt)
            {
              wordEnt+=lstEntityList.get(i+1).word+" ";
              positionEnt=positionNextEnt;
              i++;
              positionNextEnt=lstEntityList.get(i+1).position;
            }
             lstFinal.add(new ExtractNamedEntity(wordEnt,entityEnt,positionEnt));
             wordEnt="";
            }
            catch(Exception ex){}
        }
        return lstFinal;
	}