Python movie_reviews.categories方法代码示例

# 需要导入模块: from nltk.corpus import movie_reviews [as 别名]
# 或者: from nltk.corpus.movie_reviews import categories [as 别名]
def load_movie_reviews():

    # movie_reviews is a sizeable corpus to import, so only load it if we have to
    from nltk.corpus import movie_reviews
    try:
        movie_reviews.categories()
    except:
        import nltk
        print('This appears to be your first time using the NLTK Movie Reviews corpus. We will first download the necessary corpus (this is a one-time download that might take a little while')
        nltk.download('movie_reviews')
        from nltk.corpus import movie_reviews

    raw_data = []

    # NLTK's corpus is structured in an interesting way
    # first iterate through the two categories (pos and neg)
    for category in movie_reviews.categories():

        if category == 'pos':
            pretty_category_name = 'positive'
        elif category == 'neg':
            pretty_category_name = 'negative'

        # each of these categories is just fileids, so grab those
        for fileid in movie_reviews.fileids(category):

            # then each review is a NLTK class where each item in that class instance is a word
            review_words = movie_reviews.words(fileid)
            review_text = ''

            for word in review_words:
                review_text += ' ' + word

            review_dictionary = {
                'text': review_text,
                'sentiment': pretty_category_name
            }

            raw_data.append(review_dictionary)

    return raw_data 

# 需要导入模块: from nltk.corpus import movie_reviews [as 别名]
# 或者: from nltk.corpus.movie_reviews import categories [as 别名]
def getFeatures(numWordsToUse):
    # stopwords are common words that occur so frequently as to be useless for NLP
    stopWords = set(stopwords.words('english'))


    # read in all the words of each movie review, and it's associated sentiment
    reviewDocuments = []
    sentiment = []

    for category in movie_reviews.categories():
        for fileid in movie_reviews.fileids(category):
            reviewWords = movie_reviews.words(fileid)

            cleanedReview = []
            for word in reviewWords:
                if word not in stopWords:
                    cleanedReview.append(word)

            reviewDocuments.append(cleanedReview)
            if category == 'pos':
                sentiment.append(1)
            elif category == 'neg':
                sentiment.append(0)
            else:
                print 'We are not sure what this category is: ' + category

    global popularWords
    formattedReviews, sentiment, popularWords = utils.nlpFeatureEngineering(
            reviewDocuments, sentiment, 50, numWordsToUse, 'counts'
        )


    # transform list of dictionaries into a sparse matrix
    sparseFeatures = dv.fit_transform(formattedReviews)

    return sparseFeatures, sentiment