Java CoreLabel.lemma方法代碼示例

import edu.stanford.nlp.ling.CoreLabel; //導入方法依賴的package包/類
public LinkedList<String> tokenize(String text) {
    LinkedList<String> res = new LinkedList<>();
    if (text != null) {
        Annotation qaTokens = new Annotation(text);
        pipelineTokens.annotate(qaTokens);
        List<CoreMap> qssTokens = qaTokens.get(CoreAnnotations.SentencesAnnotation.class);
        for (CoreMap sentenceTokens : qssTokens) {
            ArrayList<CoreLabel> tokens = (ArrayList<CoreLabel>) sentenceTokens.get(CoreAnnotations.TokensAnnotation.class);
            for (CoreLabel t : tokens) {
                String lemma = t.lemma();
                String pos = t.tag();
                if (!stopwords.contains(lemma)) {
                    String rep = representativeProvider.getRepresentative(lemma, pos);
                    if (!stopwords.contains(rep)) {
                        res.add(rep);
                    }
                }
            }
        }
    }
    return res;
}
 

import edu.stanford.nlp.ling.CoreLabel; //導入方法依賴的package包/類
/** Given a phrase, if it contains NERs, make a dictionary minimization around them **/
public void namedEntityDictionaryMinimization(List<CoreMap> remWords, List<CoreMap> matchWords){
    // If (.* DT+ [RB|JJ]* NER+ .*) => drop (DT+)
    this.tPattern = TokenSequencePattern.compile(REGEX.T_RB_JJ_NER);
    this.tMatcher = tPattern.getMatcher(this.phrase.getWordCoreLabelList());
    while (this.tMatcher.find()){         
        matchWords = tMatcher.groupNodes();
        
        for (CoreMap cm: matchWords){
            CoreLabel cl = new CoreLabel(cm);
            if (cl.lemma() == null) cl.setLemma(cl.word());
            
            // Check if the word is DT, drop it
            if ((CoreNLPUtils.isAdj(cl.tag()) || CoreNLPUtils.isAdverb(cl.tag())) 
                    && cl.ner().equals(NE_TYPE.NO_NER)){
                remWords.add(cm);   
            }
        }
        
        // Drop the words not found in dict. 
        this.dropWordsNotFoundInDict(matchWords, remWords);
    }
    
    // Do the safe minimization
    this.namedEntitySafeMinimization(remWords, matchWords);
}
 

import edu.stanford.nlp.ling.CoreLabel; //導入方法依賴的package包/類
@Override public void addingContentWord(CoreLabel token) {
    super.addingContentWord(token);

    token.set(ReadabilityAnnotations.DifficultyLevelAnnotation.class, 4);
    String lemma = token.lemma();
    if (model.getLevel3Lemmas().contains(lemma)) {
        level3WordSize++;
        token.set(ReadabilityAnnotations.DifficultyLevelAnnotation.class, 3);
    }
    if (model.getLevel2Lemmas().contains(lemma)) {
        level2WordSize++;
        token.set(ReadabilityAnnotations.DifficultyLevelAnnotation.class, 2);
    }
    if (model.getLevel1Lemmas().contains(lemma)) {
        level1WordSize++;
        token.set(ReadabilityAnnotations.DifficultyLevelAnnotation.class, 1);
    }
}
 

import edu.stanford.nlp.ling.CoreLabel; //導入方法依賴的package包/類
public LinkedList<String> simpleTokenization(String text) {
    LinkedList<String> res = new LinkedList<>();
    if (text != null) {
        Annotation qaTokens = new Annotation(text);
        pipelineTokens.annotate(qaTokens);
        List<CoreMap> qssTokens = qaTokens.get(CoreAnnotations.SentencesAnnotation.class);
        for (CoreMap sentenceTokens : qssTokens) {
            ArrayList<CoreLabel> tokens = (ArrayList<CoreLabel>) sentenceTokens.get(CoreAnnotations.TokensAnnotation.class);
            for (CoreLabel t : tokens) {
                String lemma = t.lemma();
                String pos = t.tag();
                if ((pos.startsWith("N") || pos.startsWith("V")) && !stopwords.contains(lemma)) {
                    res.add(lemma);
                }
            }
        }
    }
    return res;
}
 

import edu.stanford.nlp.ling.CoreLabel; //導入方法依賴的package包/類
/** Given a phrase, if it contains a verb phrase, make a verb phrase safe minimization **/
public void verbPhraseSafeMinimization(List<CoreMap> remWords, List<CoreMap> matchWords){
    // Flags for checking certain conditions
    boolean isAdverb;
    boolean isNotNER;
    boolean containsNEG;
    
    // If the relation starts with a RB+ VB+, drop RB+
    this.tPattern = TokenSequencePattern.compile(REGEX.T_RB_VB);
    this.tMatcher = tPattern.getMatcher(this.phrase.getWordCoreLabelList());
    while (this.tMatcher.find()){   
        matchWords = tMatcher.groupNodes();
        
        for (CoreMap cm: matchWords){
            CoreLabel cl = new CoreLabel(cm);
            if (cl.lemma() == null) cl.setLemma(cl.word());
            
            isAdverb = CoreNLPUtils.isAdverb(cl.tag());
            isNotNER = cl.ner().equals(NE_TYPE.NO_NER);
            containsNEG = Polarity.NEG_WORDS.contains(cl.lemma().toLowerCase());
            
            // Check if the word is RB which is not a NER
            if (isAdverb && isNotNER && !containsNEG){
                remWords.add(cm);   
            }
        }
        this.dropWords(remWords, matchWords);
    }
}
 

import edu.stanford.nlp.ling.CoreLabel; //導入方法依賴的package包/類
@Override public void addingContentWord(CoreLabel token) {
        super.addingContentWord(token);

        token.set(ReadabilityAnnotations.DifficultyLevelAnnotation.class, 4);
        String lemma = token.lemma();
        if (model.getLevel3Lemmas().contains(lemma)) {
            level3WordSize++;
            token.set(ReadabilityAnnotations.DifficultyLevelAnnotation.class, 3);
        }
        if (model.getLevel2Lemmas().contains(lemma)) {
            level2WordSize++;
            token.set(ReadabilityAnnotations.DifficultyLevelAnnotation.class, 2);
        }
        if (model.getLevel1Lemmas().contains(lemma)) {
            level1WordSize++;
            token.set(ReadabilityAnnotations.DifficultyLevelAnnotation.class, 1);
        }
//        System.out.println("Adding content word (lemma): " + lemma);
//        System.out.println(model.getLevel1Lemmas().contains(lemma));
//        System.out.println(model.getLevel2Lemmas().contains(lemma));
//        System.out.println(model.getLevel3Lemmas().contains(lemma));
//        System.out.println();

//        HashMap<Integer, HashMultimap<String, String>> easyWords = model.getEasyWords();
//        String simplePos = getGenericPos(token.get(CoreAnnotations.PartOfSpeechAnnotation.class));
//        String lemma = token.get(CoreAnnotations.LemmaAnnotation.class);
//
//        if (easyWords.get(1).get(simplePos).contains(lemma)) {
//            level1WordSize++;
//        }
//        if (easyWords.get(2).get(simplePos).contains(lemma)) {
//            level2WordSize++;
//        }
//        if (easyWords.get(3).get(simplePos).contains(lemma)) {
//            level3WordSize++;
//        }
    }
 

import edu.stanford.nlp.ling.CoreLabel; //導入方法依賴的package包/類
public LinkedList<LinkedList<String>> tokenizeBySentence(String text, LinkedList<int[]> boundaries) {
    LinkedList<LinkedList<String>> res = new LinkedList<>();
    if (text != null) {
        Annotation qaTokens = new Annotation(text);
        pipelineTokens.annotate(qaTokens);
        List<CoreMap> qssTokens = qaTokens.get(CoreAnnotations.SentencesAnnotation.class);
        for (CoreMap sentenceTokens : qssTokens) {
            ArrayList<CoreLabel> tokens = (ArrayList<CoreLabel>) sentenceTokens.get(CoreAnnotations.TokensAnnotation.class);
            LinkedList<String> sentence=new LinkedList<>();
            boolean first=true;
            int[] bounds=new int[2];
            CoreLabel last=null;
            for (CoreLabel t : tokens) {
                if (first) {
                    bounds[0]=t.beginPosition();
                    first=false;
                }
                last=t;
                String lemma = t.lemma();
                String pos = t.tag();
                if (!stopwords.contains(lemma)) {
                    String rep = representativeProvider.getRepresentative(lemma, pos);
                    if (!stopwords.contains(rep)) {
                        sentence.add(rep);
                    }
                }
            }
            bounds[1]=last.endPosition();
            if (sentence.size()>0) {
                res.add(sentence);
                boundaries.add(bounds);
            }
        }
    }
    return res;
}
 

import edu.stanford.nlp.ling.CoreLabel; //導入方法依賴的package包/類
/** Given a phrase, if there is (DT* VB+ NN+), remove (DT* VB+) */
public void removeVerbsBeforeNouns(List<CoreMap> remWords, List<CoreMap> matchWords){
    // Flags for checking certain conditions
    boolean isDT;
    boolean isVerb;
    boolean isNotNER;
    boolean containsNEG;
    boolean hasDT = false;
    
    this.tPattern = TokenSequencePattern.compile(REGEX.T_DT_VB_NN_END);
    this.tMatcher = tPattern.getMatcher(this.phrase.getWordCoreLabelList());
    while (this.tMatcher.find()){
        matchWords = tMatcher.groupNodes();
        
        for (CoreMap cm: matchWords){
            CoreLabel cl = new CoreLabel(cm);
            if (cl.lemma() == null) cl.setLemma(cl.word());
            // Check if the word is a determiner, no ner and the first word in the phrase
            isDT = cl.tag().equals(POS_TAG.DT);
            isNotNER = cl.ner().equals(NE_TYPE.NO_NER);
            containsNEG = Polarity.NEG_WORDS.contains(cl.lemma().toLowerCase());
            isVerb = CoreNLPUtils.isVerb(cl.tag());
            
            if (isDT && isNotNER && !containsNEG){
                if (cl.index() == this.phrase.getWordCoreLabelList().get(0).index()){
                    remWords.add(cm);
                    hasDT = true;
                } else break;
            }
            // Check if the word is a verb, no ner
            else if (isVerb && isNotNER && !containsNEG){
                // If it's not preceded by DT, check if it's the first word in the phrase
                if (!hasDT) {
                    if (cl.index() == this.phrase.getWordCoreLabelList().get(0).index()){
                        if (!this.sg.hasChildren(new IndexedWord(cl))){
                            remWords.add(cm);
                        }
                    } else break;
                } else {
                    if (!this.sg.hasChildren(new IndexedWord(cl))){
                        remWords.add(cm);
                    }
                }
            }
        }
        
        // If the multi-word expression is found in the dictionary - don't drop it
        if (this.isCoreMapListInDictionary(matchWords)){
            matchWords.clear();
            remWords.clear();
            continue;
        }
        
        this.dropWords(remWords, matchWords);
    }
}