当前位置: 首页>>代码示例>>Python>>正文


Python nltk.wordpunct_tokenize方法代码示例

本文整理汇总了Python中nltk.wordpunct_tokenize方法的典型用法代码示例。如果您正苦于以下问题:Python nltk.wordpunct_tokenize方法的具体用法?Python nltk.wordpunct_tokenize怎么用?Python nltk.wordpunct_tokenize使用的例子?那么, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在nltk的用法示例。


在下文中一共展示了nltk.wordpunct_tokenize方法的15个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Python代码示例。

示例1: tokenize

# 需要导入模块: import nltk [as 别名]
# 或者: from nltk import wordpunct_tokenize [as 别名]
def tokenize(self, text):
        """
        Returns a list of individual tokens from the text utilizing NLTK's
        tokenize built in utility (far better than split on space). It also
        removes any stopwords and punctuation from the text, as well as
        ensure that every token is normalized.

        For now, token = word as in bag of words (the feature we're using).
        """
        for token in wordpunct_tokenize(text):
            token = self.normalize(token)
            if token in self.punctuation: continue
            if token in self.stopwords: continue
            yield token 
开发者ID:georgetown-analytics,项目名称:product-classifier,代码行数:16,代码来源:features.py

示例2: parse

# 需要导入模块: import nltk [as 别名]
# 或者: from nltk import wordpunct_tokenize [as 别名]
def parse(sent):
    parser = nltk.ChartParser(grammar)
    tokens = nltk.wordpunct_tokenize(sent)
    return parser.parse(tokens) 
开发者ID:foxbook,项目名称:atap,代码行数:6,代码来源:parse.py

示例3: recommend

# 需要导入模块: import nltk [as 别名]
# 或者: from nltk import wordpunct_tokenize [as 别名]
def recommend(self, terms):
        """
        Given input list of ingredient terms,
        return the k closest matching recipes.

        :param terms: list of strings
        :return: list of document indices of documents
        """
        vect_doc = self.vect.transform(wordpunct_tokenize(terms))
        distance_matches = self.knn.transform(vect_doc)
        # the result is a list with a 2-tuple of arrays
        matches = distance_matches[0][1][0]
        # the matches are the indices of documents
        return matches 
开发者ID:foxbook,项目名称:atap,代码行数:16,代码来源:recommender.py

示例4: query

# 需要导入模块: import nltk [as 别名]
# 或者: from nltk import wordpunct_tokenize [as 别名]
def query(self, terms):
        """
        Given input list of ingredient terms,
        return the k closest matching recipes.

        :param terms: list of strings
        :return: list of document indices of documents
        """
        vect_doc = self.transformer.named_steps['transform'].fit_transform(
            wordpunct_tokenize(terms)
        )
        dists, inds = self.tree.query(vect_doc, k=self.k)
        return inds[0] 
开发者ID:foxbook,项目名称:atap,代码行数:15,代码来源:recommender.py

示例5: words

# 需要导入模块: import nltk [as 别名]
# 或者: from nltk import wordpunct_tokenize [as 别名]
def words(self, fileids=None, categories=None):
        """
        Uses the built in word tokenizer to extract tokens from sentences.
        Note that this method uses BeautifulSoup to parse HTML content.
        """
        for sentence in self.sents(fileids, categories):
            for token in wordpunct_tokenize(sentence):
                yield token 
开发者ID:foxbook,项目名称:atap,代码行数:10,代码来源:reader.py

示例6: tokenize

# 需要导入模块: import nltk [as 别名]
# 或者: from nltk import wordpunct_tokenize [as 别名]
def tokenize(self, fileids=None, categories=None):
        """
        Segments, tokenizes, and tags a document in the corpus.
        """
        for paragraph in self.corpus.paras(fileids=fileid):
            yield [
                pos_tag(nltk.wordpunct_tokenize(sent))
                for sent in nltk.sent_tokenize(paragraph)
            ] 
开发者ID:foxbook,项目名称:atap,代码行数:11,代码来源:reader.py

示例7: tokenize

# 需要导入模块: import nltk [as 别名]
# 或者: from nltk import wordpunct_tokenize [as 别名]
def tokenize(self, fileids=None, categories=None):
        """
        Segments, tokenizes, and tags a document in the corpus.
        """
        for paragraph in self.paras(fileids=fileids):
            yield [
                pos_tag(wordpunct_tokenize(sent))
                for sent in sent_tokenize(paragraph)
            ] 
开发者ID:foxbook,项目名称:atap,代码行数:11,代码来源:reader.py

示例8: describe

# 需要导入模块: import nltk [as 别名]
# 或者: from nltk import wordpunct_tokenize [as 别名]
def describe(self, fileids=None, categories=None):
        """
        Performs a single pass of the corpus and
        returns a dictionary with a variety of metrics
        concerning the state of the corpus.
        """
        started = time.time()

        # Structures to perform counting.
        counts  = nltk.FreqDist()
        tokens  = nltk.FreqDist()

        # Perform single pass over paragraphs, tokenize and count
        for para in self.paras(fileids, categories):
            counts['paras'] += 1

            for sent in sent_tokenize(para):
                counts['sents'] += 1

                for word in wordpunct_tokenize(sent):
                    counts['words'] += 1
                    tokens[word] += 1

        # Compute the number of files and categories in the corpus
        n_fileids = len(self.resolve(fileids, categories) or self.fileids())
        n_topics  = len(self.categories(self.resolve(fileids, categories)))

        # Return data structure with information
        return {
            'files':  n_fileids,
            'topics': n_topics,
            'paras':  counts['paras'],
            'sents':  counts['sents'],
            'words':  counts['words'],
            'vocab':  len(tokens),
            'lexdiv': float(counts['words']) / float(len(tokens)),
            'ppdoc':  float(counts['paras']) / float(n_fileids),
            'sppar':  float(counts['sents']) / float(counts['paras']),
            'secs':   time.time() - started,
        } 
开发者ID:foxbook,项目名称:atap,代码行数:42,代码来源:reader.py

示例9: tokenize

# 需要导入模块: import nltk [as 别名]
# 或者: from nltk import wordpunct_tokenize [as 别名]
def tokenize(self, fileid):
        """
        Segments, tokenizes, and tags a document in the corpus. Returns a
        generator of paragraphs, which are lists of sentences, which in turn
        are lists of part of speech tagged words.
        """
        for paragraph in self.corpus.paras(fileids=fileid):
            yield [
                pos_tag(wordpunct_tokenize(sent))
                for sent in sent_tokenize(paragraph)
            ] 
开发者ID:foxbook,项目名称:atap,代码行数:13,代码来源:preprocess.py

示例10: words

# 需要导入模块: import nltk [as 别名]
# 或者: from nltk import wordpunct_tokenize [as 别名]
def words(self):
        """
        Returns a generator of words.
        """
        for sent in self.sents():
            for word in nltk.wordpunct_tokenize(sent):
                yield word 
开发者ID:foxbook,项目名称:atap,代码行数:9,代码来源:am_reader.py

示例11: tagged_tokens

# 需要导入模块: import nltk [as 别名]
# 或者: from nltk import wordpunct_tokenize [as 别名]
def tagged_tokens(self):
        for sent in self.sents():
            for word in nltk.wordpunct_tokenize(sent):
                yield nltk.pos_tag(word) 
开发者ID:foxbook,项目名称:atap,代码行数:6,代码来源:reader.py

示例12: calculate_language_scores

# 需要导入模块: import nltk [as 别名]
# 或者: from nltk import wordpunct_tokenize [as 别名]
def calculate_language_scores(text):
    """
    Calculate probability of given text to be written in several languages and
    return a dictionary that looks like {'french': 2, 'spanish': 4, 'english': 0}.

    :param text: Text to analyze.
    :type text: str

    :return: Dictionary with languages and unique stopwords seen in analyzed text.
    :rtype: dict(str -> int)

    :raises: TypeError
    """
    if not isinstance(text, basestring):
        raise TypeError("Expected basestring, got '%s' instead" % type(text))
    if not text:
        return {}

    languages_ratios = {}

    # Split the text into separate tokens, using natural language punctuation signs.
    tokens = wordpunct_tokenize(text)
    tokenized_words = [word.lower() for word in tokens]

    for language in stopwords.fileids():
        stopwords_set = set(stopwords.words(language))
        words_set = set(tokenized_words)
        common_elements = words_set.intersection(stopwords_set)
        languages_ratios[language] = len(common_elements)  # language "score"

    return languages_ratios


#------------------------------------------------------------------------------ 
开发者ID:blackye,项目名称:luscan-devel,代码行数:36,代码来源:natural_language.py

示例13: normalize

# 需要导入模块: import nltk [as 别名]
# 或者: from nltk import wordpunct_tokenize [as 别名]
def normalize(sent):
    return wordpunct_tokenize(sent.lower()) 
开发者ID:edward-zhu,项目名称:dialog,代码行数:4,代码来源:cluster.py

示例14: tokenize

# 需要导入模块: import nltk [as 别名]
# 或者: from nltk import wordpunct_tokenize [as 别名]
def tokenize(sent):
    tokens = tokenizer.tokenize(sent)
    ret = []
    for t in tokens:
        if '<' not in t:
            ret.extend(wordpunct_tokenize(t))
        else:
            ret.append(t)
    return ret 
开发者ID:edward-zhu,项目名称:dialog,代码行数:11,代码来源:utils.py

示例15: tokenize_and_normalize

# 需要导入模块: import nltk [as 别名]
# 或者: from nltk import wordpunct_tokenize [as 别名]
def tokenize_and_normalize(s):
    """Tokenize and normalize string."""
    token_list = []
    tokens = wordpunct_tokenize(s.lower())
    token_list.extend([x for x in tokens if not re.fullmatch('[' + string.punctuation + ']+', x)])
    return token_list 
开发者ID:wasiahmad,项目名称:transferable_sent2vec,代码行数:8,代码来源:helper.py


注:本文中的nltk.wordpunct_tokenize方法示例由纯净天空整理自Github/MSDocs等开源代码及文档管理平台,相关代码片段筛选自各路编程大神贡献的开源项目,源码版权归原作者所有,传播和使用请参考对应项目的License;未经允许,请勿转载。