Python WordNetLemmatizer.stem方法代码示例

# 需要导入模块: from nltk.stem.wordnet import WordNetLemmatizer [as 别名]
# 或者: from nltk.stem.wordnet.WordNetLemmatizer import stem [as 别名]
	def lemmatize(self, token, v=1):
		if v == self.STEMMER_WORDNET_ENGLISH:
			from nltk.stem.wordnet import WordNetLemmatizer
			lmtzr = WordNetLemmatizer()
			return lmtzr.lemmatize(token)

		if v == self.STEMMER_SNOWBALL_ENGLISH:
			from nltk import stem
			lmtzr =stem.snowball.EnglishStemmer()
			return lmtzr.stem(token)

		if v == self.STEMMER_LANCASTER_ENGLISH:
			from nltk.stem.lancaster import LancasterStemmer
			lmtzr = LancasterStemmer()
			return lmtzr.stem(token)

		if v == self.STEMMER_PORTER_ENGLISH:
			from nltk.stem.porter import PorterStemmer
			lmtzr = PorterStemmer()
			return lmtzr.stem(token)

# 需要导入模块: from nltk.stem.wordnet import WordNetLemmatizer [as 别名]
# 或者: from nltk.stem.wordnet.WordNetLemmatizer import stem [as 别名]
class RTEInferenceTagger(object):
    """
    Predict whether a hypothesis can be inferred from a text, 
    based on the degree of word overlap.
    """
    def __init__(self, threshold=33, stop=True):
        self.threshold = threshold
        self.stop = stop
        self.stopwords = set(['a', 'the', 'it', 'they', 'of', 'in', 'is', 'are', 'were', 'and'])
        self.stemmer = WordNetLemmatizer()
    
    def tag(self, rtepair, verbose=False):
        """
        Tag a RTEPair as to whether the hypothesis can be inferred from the text.
        """
        return self.tag_sentences(rtepair.text, rtepair.hyp )

    def tag_sentences(self, text, hyp, verbose=False):
        """
        Tag a RTEPair as to whether the hypothesis can be inferred from the text.
        """
        glueclass = DrtGlue()
        text_drs_list = glueclass.parse_to_meaning(text)
        if text_drs_list:
            text_ex = text_drs_list[0].simplify().toFol()
        else:
            if verbose: print 'ERROR: No readings were generated for the Text'
        
        hyp_drs_list = glueclass.parse_to_meaning(hyp)
        if hyp_drs_list:
            hyp_ex = hyp_drs_list[0].simplify().toFol()
        else:
            if verbose: print 'ERROR: No readings were generated for the Hypothesis'

        #1. proof T -> H
        #2. proof (BK & T) -> H
        #3. proof :(BK & T)
        #4. proof :(BK & T & H)
        #5. satisfy BK & T
        #6. satisfy BK & T & H
            
        result = inference.Prover9().prove(hyp_ex, [text_ex])
        if verbose: print 'prove: T -> H: %s' % result
        
        if not result:
            bk = self._generate_BK(text, hyp, verbose)
            bk_exs = [bk_pair[0] for bk_pair in bk]
            
            if verbose: 
                print 'Generated Background Knowledge:'
                for bk_ex in bk_exs:
                    print bk_ex
                
            result = inference.Prover9().prove(hyp_ex, [text_ex]+bk_exs)
            if verbose: print 'prove: (T & BK) -> H: %s' % result
            
            if not result:
                consistent = self.check_consistency(bk_exs+[text_ex])                
                if verbose: print 'consistency check: (BK & T): %s' % consistent

                if consistent:
                    consistent = self.check_consistency(bk_exs+[text_ex, hyp_ex])                
                    if verbose: print 'consistency check: (BK & T & H): %s' % consistent
                    
        return result
    
    def check_consistency(self, assumptions, verbose=False):
        return inference.ParallelProverBuilderCommand(assumptions=assumptions).build_model()
        
    def _tag(self, text, hyp, verbose=False):
        self._generate_BK(text, hyp, verbose)
    
    def _generate_BK(self, text, hyp, verbose=False):
        text = word_tokenize(text)
        hyp = word_tokenize(hyp)
        
        if self.stemmer:
            textbow = set(self._stem(word) for word in text)
            hypbow = set(self._stem(word) for word in hyp)
        else:
            textbow = set(word.lower() for word in text)
            hypbow = set(word.lower() for word in hyp)
        
        if verbose:
            print 'textbow: %s' % textbow
            print 'hypbow: %s' % hypbow
        
        if self.stop:
            textbow = textbow - self.stopwords
            hypbow = hypbow - self.stopwords

        bk = []
        fullbow = textbow|hypbow
        for word_text in fullbow:
            pos = None
            if word_text in wordnet.N:
                bk.extend(self._generate_BK_word(word_text, wordnet.N, fullbow))
            if word_text in wordnet.V:
                bk.extend(self._generate_BK_word(word_text, wordnet.V, fullbow))
            if word_text in wordnet.ADJ:
#.........这里部分代码省略.........