Python porter.PorterStemmer方法代码示例

# 需要导入模块: from nltk.stem import porter [as 别名]
# 或者: from nltk.stem.porter import PorterStemmer [as 别名]
def test_dictionary_match(doc_setup):
    """Test DictionaryMatch matcher."""
    doc = doc_setup
    space = MentionNgrams(n_min=1, n_max=1)

    # Test with a list of str
    matcher = DictionaryMatch(d=["this"])
    assert set(tc.get_span() for tc in matcher.apply(space.apply(doc))) == {"This"}

    # Test without a dictionary
    with pytest.raises(Exception):
        DictionaryMatch()

    # TODO: test with plural words
    matcher = DictionaryMatch(d=["is"], stemmer=PorterStemmer())
    assert set(tc.get_span() for tc in matcher.apply(space.apply(doc))) == {"is"}

    # Test if matcher raises an error when _f is given non-TemporarySpanMention
    matcher = DictionaryMatch(d=["this"])
    with pytest.raises(ValueError):
        list(matcher.apply(doc.sentences[0].words)) 

# 需要导入模块: from nltk.stem import porter [as 别名]
# 或者: from nltk.stem.porter import PorterStemmer [as 别名]
def load(tdb):
    # load the tasks and arxiv metadata
    stemmer = PorterStemmer()

    tdb.load_tasks("data/tasks/nlpprogress.json")
    tdb.load_synonyms(["data/tasks/synonyms.csv"])
    arxiv = serialization.load(
        "data/arxiv_aclweb.json.gz", fmt=serialization.Format.json_gz
    )

    for a in arxiv:
        if a["abstract"] is None:
            a["abstract"] = ""

    # require and normalise arxiv titles
    arxiv = [a for a in arxiv if "title" in a and a["title"] is not None]
    for a in arxiv:
        a["title"] = re.sub(" +", " ", a["title"].replace("\n", " "))
        a["title_lower"] = a["title"].lower()
        a["abstract_lower"] = a["abstract"].lower()
        a["title_stem"] = stemmer.stem(a["title"])
        a["abstract_stem"] = stemmer.stem(a["abstract"])

    return arxiv 

# 需要导入模块: from nltk.stem import porter [as 别名]
# 或者: from nltk.stem.porter import PorterStemmer [as 别名]
def tiny_tokenize(text, stem=False, stop_words=[]):
    words = []
    for token in wordpunct_tokenize(re.sub('[%s]' % re.escape(string.punctuation), ' ', \
            text.decode(encoding='UTF-8', errors='ignore'))):
        if not token.isdigit() and not token in stop_words:
            if stem:
                try:
                    w = EnglishStemmer().stem(token)
                except Exception as e:
                    w = token
            else:
                w = token
            words.append(w)

    return words

    # return [EnglishStemmer().stem(token) if stem else token for token in wordpunct_tokenize(
    #                     re.sub('[%s]' % re.escape(string.punctuation), ' ', text.decode(encoding='UTF-8', errors='ignore'))) if
    #                     not token.isdigit() and not token in stop_words] 

# 需要导入模块: from nltk.stem import porter [as 别名]
# 或者: from nltk.stem.porter import PorterStemmer [as 别名]
def __init__(self,paragraphs,removeStopWord = False,useStemmer = False):
        self.idf = {}               # dict to store IDF for words in paragraph
        self.paragraphInfo = {}     # structure to store paragraphVector
        self.paragraphs = paragraphs
        self.totalParas = len(paragraphs)
        self.stopwords = stopwords.words('english')
        self.removeStopWord = removeStopWord
        self.useStemmer = useStemmer
        self.vData = None
        self.stem = lambda k:k.lower()
        if(useStemmer):
            ps = PorterStemmer()
            self.stem = ps.stem
            
        # Initialize
        self.computeTFIDF()
        
    # Return term frequency for Paragraph
    # Input:
    #       paragraph(str): Paragraph as a whole in string format
    # Output:
    #       wordFrequence(dict) : Dictionary of word and term frequency 

# 需要导入模块: from nltk.stem import porter [as 别名]
# 或者: from nltk.stem.porter import PorterStemmer [as 别名]
def sim_sentence(self, queryVector, sentence):
        sentToken = word_tokenize(sentence)
        ps = PorterStemmer()
        for index in range(0,len(sentToken)):
            sentToken[index] = ps.stem(sentToken[index])
        sim = 0
        for word in queryVector.keys():
            w = ps.stem(word)
            if w in sentToken:
                sim += 1
        return sim/(len(sentToken)*len(queryVector.keys()))
    
    # Get Named Entity from the sentence in form of PERSON, GPE, & ORGANIZATION
    # Input:
    #       answers(list)       : List of potential sentence containing answer
    # Output:
    #       chunks(list)        : List of tuple with entity and name in ranked 
    #                             order 

# 需要导入模块: from nltk.stem import porter [as 别名]
# 或者: from nltk.stem.porter import PorterStemmer [as 别名]
def stem_match(hypothesis, reference, stemmer = PorterStemmer()):
    """
    Stems each word and matches them in hypothesis and reference 
    and returns a word mapping between hypothesis and reference

    :param hypothesis:
    :type hypothesis:
    :param reference:
    :type reference:
    :param stemmer: nltk.stem.api.StemmerI object (default PorterStemmer())
    :type stemmer: nltk.stem.api.StemmerI or any class that 
                   implements a stem method
    :return: enumerated matched tuples, enumerated unmatched hypothesis tuples, 
             enumerated unmatched reference tuples
    :rtype: list of 2D tuples, list of 2D tuples,  list of 2D tuples
    """
    enum_hypothesis_list, enum_reference_list = _generate_enums(hypothesis, reference)
    return _enum_stem_match(enum_hypothesis_list, enum_reference_list, stemmer = stemmer) 

# 需要导入模块: from nltk.stem import porter [as 别名]
# 或者: from nltk.stem.porter import PorterStemmer [as 别名]
def allign_words(hypothesis, reference, stemmer = PorterStemmer(), wordnet = wordnet):
    """
    Aligns/matches words in the hypothesis to reference by sequentially 
    applying exact match, stemmed match and wordnet based synonym match. 
    In case there are multiple matches the match which has the least number
    of crossing is chosen.

    :param hypothesis: hypothesis string
    :param reference: reference string
    :param stemmer: nltk.stem.api.StemmerI object (default PorterStemmer())
    :type stemmer: nltk.stem.api.StemmerI or any class that implements a stem method
    :param wordnet: a wordnet corpus reader object (default nltk.corpus.wordnet)
    :type wordnet: WordNetCorpusReader
    :return: sorted list of matched tuples, unmatched hypothesis list, unmatched reference list
    :rtype: list of tuples, list of tuples, list of tuples
    """
    enum_hypothesis_list, enum_reference_list = _generate_enums(hypothesis, reference)
    return _enum_allign_words(enum_hypothesis_list, enum_reference_list, stemmer= stemmer,
                             wordnet= wordnet) 

# 需要导入模块: from nltk.stem import porter [as 别名]
# 或者: from nltk.stem.porter import PorterStemmer [as 别名]
def __init__(self, ignore_stopwords=False):
        _LanguageSpecificStemmer.__init__(self, ignore_stopwords)
        porter.PorterStemmer.__init__(self) 

# 需要导入模块: from nltk.stem import porter [as 别名]
# 或者: from nltk.stem.porter import PorterStemmer [as 别名]
def clean(s):
    tokens = nltk.word_tokenize(s.lower())
    tokens_clean = [token for token in tokens if token not in stopwords.words('english')]
    tokens_stemmed = [PorterStemmer().stem(token) for token in tokens_clean]
    return tokens_stemmed 

# 需要导入模块: from nltk.stem import porter [as 别名]
# 或者: from nltk.stem.porter import PorterStemmer [as 别名]
def __lemmatizer(self, docs):
		output = []
		for doc in docs:
			stemmer = PorterStemmer()
			texts = [stemmer.stem(i) for i in doc]
			output.append(texts)
		return output 

# 需要导入模块: from nltk.stem import porter [as 别名]
# 或者: from nltk.stem.porter import PorterStemmer [as 别名]
def addtoVocab(self, words):
        #stemmer = PorterStemmer()
        w_list = self.removeStopWords(words)
        for word in w_list:
            self.vocabulary[word] += 1
        return w_list 

# 需要导入模块: from nltk.stem import porter [as 别名]
# 或者: from nltk.stem.porter import PorterStemmer [as 别名]
def get_stemmed_combined_reviews(indeed_reviews_db, glassdoor_reviews_db):
    combined = get_combined_reviews(indeed_reviews_db, glassdoor_reviews_db)

    stemmer = PorterStemmer()
    stemmed_reviews = []
    for review in combined:
        stemmed_reviews.append(' '.join([stemmer.stem(word) for sent in sent_tokenize(review) for word in word_tokenize(sent.lower())]))

    return stemmed_reviews 

# 需要导入模块: from nltk.stem import porter [as 别名]
# 或者: from nltk.stem.porter import PorterStemmer [as 别名]
def get_stemmed_separate(indeed_reviews_db, glassdoor_reviews_db):
    separate = get_separate_reviews(indeed_reviews_db, glassdoor_reviews_db)
    stemmer = PorterStemmer()
    stemmed_reviews = []
    for review in separate:
        stemmed_reviews.append(' '.join([stemmer.stem(word) for sent in sent_tokenize(review) for word in word_tokenize(sent.lower())]))
    return stemmed_reviews 

# 需要导入模块: from nltk.stem import porter [as 别名]
# 或者: from nltk.stem.porter import PorterStemmer [as 别名]
def test_do_not_use_stemmer_when_UnicodeDecodeError():
    """Test DictionaryMatch when stemmer causes UnicodeDecodeError."""
    stemmer = PorterStemmer()
    matcher = DictionaryMatch(d=["is"], stemmer=stemmer)
    # _stem(w) should return a word stem.
    assert matcher._stem("caresses") == "caress"

    stemmer.stem = Mock(
        side_effect=UnicodeDecodeError("dummycodec", b"\x00\x00", 1, 2, "Dummy  !")
    )
    matcher = DictionaryMatch(d=["is"], stemmer=stemmer)
    # _stem(w) should return w as stemmer.stem raises UnicodeDecodeError.
    assert matcher._stem("caresses") == "caresses" 

# 需要导入模块: from nltk.stem import porter [as 别名]
# 或者: from nltk.stem.porter import PorterStemmer [as 别名]
def tiny_tokenize_xml(text, stem=False, stop_words=[]):
    return [EnglishStemmer().stem(token) if stem else token for token in wordpunct_tokenize(
                        re.sub('[%s]' % re.escape(string.punctuation), ' ', text.encode(encoding='ascii', errors='ignore'))) if
                        not token.isdigit() and not token in stop_words]