本文整理汇总了Python中nltk.sentiment.SentimentAnalyzer类的典型用法代码示例。如果您正苦于以下问题:Python SentimentAnalyzer类的具体用法?Python SentimentAnalyzer怎么用?Python SentimentAnalyzer使用的例子?那么恭喜您, 这里精选的类代码示例或许可以为您提供帮助。
在下文中一共展示了SentimentAnalyzer类的15个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: sentiment_analysis
def sentiment_analysis(self, testing_data, training_data=None):
if training_data is None:
training_data = self.training_data
## Apply sentiment analysis to data to extract new "features"
# Initialize sentiment analyzer object
sentiment_analyzer = SentimentAnalyzer()
# Mark all negative words in training data, using existing list of negative words
all_negative_words = sentiment_analyzer.all_words([mark_negation(data) for data in training_data])
unigram_features = sentiment_analyzer.unigram_word_feats(all_negative_words, min_freq=4)
len(unigram_features)
sentiment_analyzer.add_feat_extractor(extract_unigram_feats, unigrams=unigram_features)
training_final = sentiment_analyzer.apply_features(training_data)
testing_final = sentiment_analyzer.apply_features(testing_data)
## Traing model and test
model = NaiveBayesClassifier.train
classifer = sentiment_analyzer.train(model, training_final)
for key, value in sorted(sentiment_analyzer.evaluate(testing_final).items()):
print ("{0}: {1}".format(key, value))
示例2: demo_subjectivity
def demo_subjectivity(trainer, save_analyzer=False, n_instances=None, output=None):
"""
Train and test a classifier on instances of the Subjective Dataset by Pang and
Lee. The dataset is made of 5000 subjective and 5000 objective sentences.
All tokens (words and punctuation marks) are separated by a whitespace, so
we use the basic WhitespaceTokenizer to parse the data.
:param trainer: `train` method of a classifier.
:param save_analyzer: if `True`, store the SentimentAnalyzer in a pickle file.
:param n_instances: the number of total sentences that have to be used for
training and testing. Sentences will be equally split between positive
and negative.
:param output: the output file where results have to be reported.
"""
from nltk.sentiment import SentimentAnalyzer
from nltk.corpus import subjectivity
if n_instances is not None:
n_instances = int(n_instances/2)
subj_docs = [(sent, 'subj') for sent in subjectivity.sents(categories='subj')[:n_instances]]
obj_docs = [(sent, 'obj') for sent in subjectivity.sents(categories='obj')[:n_instances]]
# We separately split subjective and objective instances to keep a balanced
# uniform class distribution in both train and test sets.
train_subj_docs, test_subj_docs = split_train_test(subj_docs)
train_obj_docs, test_obj_docs = split_train_test(obj_docs)
training_docs = train_subj_docs+train_obj_docs
testing_docs = test_subj_docs+test_obj_docs
sentim_analyzer = SentimentAnalyzer()
all_words_neg = sentim_analyzer.all_words([mark_negation(doc) for doc in training_docs])
# Add simple unigram word features handling negation
unigram_feats = sentim_analyzer.unigram_word_feats(all_words_neg, min_freq=4)
sentim_analyzer.add_feat_extractor(extract_unigram_feats, unigrams=unigram_feats)
# Apply features to obtain a feature-value representation of our datasets
training_set = sentim_analyzer.apply_features(training_docs)
test_set = sentim_analyzer.apply_features(testing_docs)
classifier = sentim_analyzer.train(trainer, training_set)
try:
classifier.show_most_informative_features()
except AttributeError:
print('Your classifier does not provide a show_most_informative_features() method.')
results = sentim_analyzer.evaluate(test_set)
if save_analyzer == True:
save_file(sentim_analyzer, 'sa_subjectivity.pickle')
if output:
extr = [f.__name__ for f in sentim_analyzer.feat_extractors]
output_markdown(output, Dataset='subjectivity', Classifier=type(classifier).__name__,
Tokenizer='WhitespaceTokenizer', Feats=extr,
Instances=n_instances, Results=results)
return sentim_analyzer
示例3: train
def train():
positive_tweets = read_tweets('positive.txt', 'positive')
negative_tweets = read_tweets('negative.txt', 'negative')
print len(positive_tweets)
print len(negative_tweets)
pos_train = positive_tweets[:len(positive_tweets)]
neg_train = negative_tweets[:len(negative_tweets)]
# pos_test = positive_tweets[len(positive_tweets)*80/100+1:]
# neg_test = negative_tweets[len(positive_tweets)*80/100+1:]
training_data = pos_train + neg_train
# test_data = pos_test + neg_test
sentim_analyzer = SentimentAnalyzer()
all_words_neg = sentim_analyzer.all_words([mark_negation(doc) for doc in training_data])
unigram_feats = sentim_analyzer.unigram_word_feats(all_words_neg, min_freq=4)
print len(unigram_feats)
sentim_analyzer.add_feat_extractor(extract_unigram_feats, unigrams=unigram_feats)
training_set = sentim_analyzer.apply_features(training_data)
# test_set = sentim_analyzer.apply_features(test_data)
# print test_set
trainer = NaiveBayesClassifier.train
sentim_analyzer.train(trainer, training_set)
# for key,value in sorted(sentim_analyzer.evaluate(test_set).items()):
# print('{0}: {1}'.format(key, value))
# print sentim_analyzer.classify(tokenize_sentence('I hate driving car at night'))
return sentim_analyzer
示例4: demo_movie_reviews
def demo_movie_reviews(trainer, n_instances=None, output=None):
"""
Train classifier on all instances of the Movie Reviews dataset.
The corpus has been preprocessed using the default sentence tokenizer and
WordPunctTokenizer.
Features are composed of:
- most frequent unigrams
:param trainer: `train` method of a classifier.
:param n_instances: the number of total reviews that have to be used for
training and testing. Reviews will be equally split between positive and
negative.
:param output: the output file where results have to be reported.
"""
from nltk.corpus import movie_reviews
from nltk.sentiment import SentimentAnalyzer
if n_instances is not None:
n_instances = int(n_instances/2)
pos_docs = [(list(movie_reviews.words(pos_id)), 'pos') for pos_id in movie_reviews.fileids('pos')[:n_instances]]
neg_docs = [(list(movie_reviews.words(neg_id)), 'neg') for neg_id in movie_reviews.fileids('neg')[:n_instances]]
# We separately split positive and negative instances to keep a balanced
# uniform class distribution in both train and test sets.
train_pos_docs, test_pos_docs = split_train_test(pos_docs)
train_neg_docs, test_neg_docs = split_train_test(neg_docs)
training_docs = train_pos_docs+train_neg_docs
testing_docs = test_pos_docs+test_neg_docs
sentim_analyzer = SentimentAnalyzer()
all_words = sentim_analyzer.all_words(training_docs)
# Add simple unigram word features
unigram_feats = sentim_analyzer.unigram_word_feats(all_words, min_freq=4)
sentim_analyzer.add_feat_extractor(extract_unigram_feats, unigrams=unigram_feats)
# Apply features to obtain a feature-value representation of our datasets
training_set = sentim_analyzer.apply_features(training_docs)
test_set = sentim_analyzer.apply_features(testing_docs)
classifier = sentim_analyzer.train(trainer, training_set)
try:
classifier.show_most_informative_features()
except AttributeError:
print('Your classifier does not provide a show_most_informative_features() method.')
results = sentim_analyzer.evaluate(test_set)
if output:
extr = [f.__name__ for f in sentim_analyzer.feat_extractors]
output_markdown(output, Dataset='Movie_reviews', Classifier=type(classifier).__name__,
Tokenizer='WordPunctTokenizer', Feats=extr, Results=results,
Instances=n_instances)
示例5: __init__
def __init__(self, sentiment_only, num_phrases_to_track=20):
# neg_phrases = filter_negative_phrases(load_csv_sentences('thoughtsandfeelings.csv'))
# pos_phrases = filter_positive_phrases(load_csv_sentences('spiritualforums.csv'))
# file_pos = open("pos_phrases.txt", 'w')
# file_neg = open("neg_phrases.txt", 'w')
# for item in pos_phrases:
# print>>file_pos, item
# for item in neg_phrases:
# print>>file_neg, item
self.recent_sentiment_scores = []
neg_file = open("ALL_neg_phrases_filtered.txt", "r")
pos_file = open("webtext_phrases_with_lots_of_words.txt", "r")
neg_phrases = neg_file.readlines()
pos_phrases = pos_file.readlines()
neg_docs = []
pos_docs = []
for phrase in neg_phrases:
neg_docs.append((phrase.split(), 'suicidal'))
for phrase in pos_phrases[:len(neg_phrases)]:
pos_docs.append((phrase.split(), 'alright'))
print len(neg_docs)
print len(pos_docs)
# negcutoff = len(neg_docs) * 3 / 4
# poscutoff = len(pos_docs) * 3 / 4
negcutoff = -200
poscutoff = -200
train_pos_docs = pos_docs[:poscutoff]
test_pos_docs = pos_docs[poscutoff:]
train_neg_docs = neg_docs[:negcutoff]
test_neg_docs = neg_docs[negcutoff:]
training_docs = train_pos_docs + train_neg_docs
testing_docs = test_pos_docs + test_neg_docs
self.sentim_analyzer = SentimentAnalyzer()
if not sentiment_only:
all_words = self.sentim_analyzer.all_words([doc for doc in training_docs])
unigram_feats = self.sentim_analyzer.unigram_word_feats(all_words, min_freq=1)
self.sentim_analyzer.add_feat_extractor(extract_unigram_feats, unigrams=unigram_feats)
self.sentim_analyzer.add_feat_extractor(vader_sentiment_feat)
# bigram_feats = self.sentim_analyzer.bigram_collocation_feats(all_words, min_freq=1)
# self.sentim_analyzer.add_feat_extractor(extract_bigram_feats, bigrams=bigram_feats)
training_set = self.sentim_analyzer.apply_features(training_docs)
test_set = self.sentim_analyzer.apply_features(testing_docs)
trainer = NaiveBayesClassifier.train
self.classifier = self.sentim_analyzer.train(trainer, training_set)
for key, value in sorted(self.sentim_analyzer.evaluate(test_set).items()):
print('{0}: {1}'.format(key, value))
self.classifier.show_most_informative_features(20)
示例6: train_model
def train_model(training):
## Apply sentiment analysis to data to extract new "features"
# Initialize sentiment analyzer object
sentiment_analyzer = SentimentAnalyzer()
# Mark all negative words in training data, using existing list of negative words
all_negative_words = sentiment_analyzer.all_words([mark_negation(data) for data in training])
unigram_features = sentiment_analyzer.unigram_word_feats(all_negative_words, min_freq=4)
len(unigram_features)
sentiment_analyzer.add_feat_extractor(extract_unigram_feats,unigrams=unigram_features)
training_final = sentiment_analyzer.apply_features(training)
return [training_final]
示例7: get_objectivity_analyzer
def get_objectivity_analyzer():
n_instances = 100
subj_docs = [(sent, 'subj') for sent in subjectivity.sents(categories='subj')[:n_instances]]
obj_docs = [(sent, 'obj') for sent in subjectivity.sents(categories='obj')[:n_instances]]
train_subj_docs = subj_docs
train_obj_docs = obj_docs
training_docs = train_subj_docs+train_obj_docs
sentim_analyzer = SentimentAnalyzer()
all_words_neg = sentim_analyzer.all_words([mark_negation(doc) for doc in training_docs])
unigram_feats = sentim_analyzer.unigram_word_feats(all_words_neg, min_freq=4)
sentim_analyzer.add_feat_extractor(extract_unigram_feats, unigrams=unigram_feats)
training_set = sentim_analyzer.apply_features(training_docs)
trainer = NaiveBayesClassifier.train
sentiment_classifier = sentim_analyzer.train(trainer, training_set)
return sentim_analyzer
示例8: __init__
def __init__(self):
self.sentim_analyzer = SentimentAnalyzer()
self.genre_dict = read_file("jsons/movie_genre_quote_dict_2.json")
context_file = "jsons/final_context.json"
movie_file = "jsons/final_movies.json"
quote_file = "jsons/final_quotes.json"
year_rating_file = "jsons/final_year_rating.json"
self.context = read_file(context_file)
self.movies = read_file(movie_file)
self.quotes = read_file(quote_file)
self.year_rating_dict = read_file(year_rating_file)
# Reincode to unicode
for i in range(len(self.context)):
self.context[i] = self.context[i].encode("utf-8").decode("utf-8")
self.movies[i] = self.movies[i].encode("utf-8").decode("utf-8")
self.quotes[i] = self.quotes[i].encode("utf-8").decode("utf-8")
self.context, self.quotes, self.movies = quote_pruner(self.context, self.quotes, self.movies)
self.inverted_index = read_file("jsons/f_inverted_index.json")
self.idf = read_file("jsons/f_idf.json")
# Initialize query tokenizer
self.tokenizer = TreebankWordTokenizer()
# Compute document norms
self.norms = compute_doc_norms(self.inverted_index, self.idf, len(self.context))
word_co_filename = "jsons/word_co.json"
word_count_filename = "jsons/word_count_dict.json"
pmi_dict_filename = "jsons/pmi_dict.json"
# Read files
self.word_co = read_file(word_co_filename)
self.word_count_dict = read_file(word_count_filename)
self.pmi_dict = read_file(pmi_dict_filename)
示例9: demo_tweets
def demo_tweets(trainer, n_instances=None, output=None):
"""
Train and test Naive Bayes classifier on 10000 tweets, tokenized using
TweetTokenizer.
Features are composed of:
- 1000 most frequent unigrams
- 100 top bigrams (using BigramAssocMeasures.pmi)
:param trainer: `train` method of a classifier.
:param n_instances: the number of total tweets that have to be used for
training and testing. Tweets will be equally split between positive and
negative.
:param output: the output file where results have to be reported.
"""
from nltk.tokenize import TweetTokenizer
from nltk.sentiment import SentimentAnalyzer
from nltk.corpus import twitter_samples, stopwords
# Different customizations for the TweetTokenizer
tokenizer = TweetTokenizer(preserve_case=False)
# tokenizer = TweetTokenizer(preserve_case=True, strip_handles=True)
# tokenizer = TweetTokenizer(reduce_len=True, strip_handles=True)
if n_instances is not None:
n_instances = int(n_instances/2)
fields = ['id', 'text']
positive_json = twitter_samples.abspath("positive_tweets.json")
positive_csv = 'positive_tweets.csv'
json2csv_preprocess(positive_json, positive_csv, fields, limit=n_instances)
negative_json = twitter_samples.abspath("negative_tweets.json")
negative_csv = 'negative_tweets.csv'
json2csv_preprocess(negative_json, negative_csv, fields, limit=n_instances)
neg_docs = parse_tweets_set(negative_csv, label='neg', word_tokenizer=tokenizer)
pos_docs = parse_tweets_set(positive_csv, label='pos', word_tokenizer=tokenizer)
# We separately split subjective and objective instances to keep a balanced
# uniform class distribution in both train and test sets.
train_pos_docs, test_pos_docs = split_train_test(pos_docs)
train_neg_docs, test_neg_docs = split_train_test(neg_docs)
training_tweets = train_pos_docs+train_neg_docs
testing_tweets = test_pos_docs+test_neg_docs
sentim_analyzer = SentimentAnalyzer()
# stopwords = stopwords.words('english')
# all_words = [word for word in sentim_analyzer.all_words(training_tweets) if word.lower() not in stopwords]
all_words = [word for word in sentim_analyzer.all_words(training_tweets)]
# Add simple unigram word features
unigram_feats = sentim_analyzer.unigram_word_feats(all_words, top_n=1000)
sentim_analyzer.add_feat_extractor(extract_unigram_feats, unigrams=unigram_feats)
# Add bigram collocation features
bigram_collocs_feats = sentim_analyzer.bigram_collocation_feats([tweet[0] for tweet in training_tweets],
top_n=100, min_freq=12)
sentim_analyzer.add_feat_extractor(extract_bigram_feats, bigrams=bigram_collocs_feats)
training_set = sentim_analyzer.apply_features(training_tweets)
test_set = sentim_analyzer.apply_features(testing_tweets)
classifier = sentim_analyzer.train(trainer, training_set)
# classifier = sentim_analyzer.train(trainer, training_set, max_iter=4)
try:
classifier.show_most_informative_features()
except AttributeError:
print('Your classifier does not provide a show_most_informative_features() method.')
results = sentim_analyzer.evaluate(test_set)
if output:
extr = [f.__name__ for f in sentim_analyzer.feat_extractors]
output_markdown(output, Dataset='labeled_tweets', Classifier=type(classifier).__name__,
Tokenizer=tokenizer.__class__.__name__, Feats=extr,
Results=results, Instances=n_instances)
示例10: SentimentAnalyzer
from nltk.classify import NaiveBayesClassifier
from nltk.corpus import subjectivity
from nltk.sentiment import SentimentAnalyzer
from nltk.sentiment.util import *
n_instances = 100
subj_docs = [(sent, 'subj') for sent in subjectivity.sents(categories='subj')[:n_instances]]
obj_docs = [(sent, 'obj') for sent in subjectivity.sents(categories='obj')[:n_instances]]
train_subj_docs = subj_docs[:80]
test_subj_docs = subj_docs[80:100]
train_obj_docs = obj_docs[:80]
test_obj_docs = obj_docs[80:100]
training_docs = train_subj_docs+train_obj_docs
testing_docs = test_subj_docs+test_obj_docs
sentim_analyzer = SentimentAnalyzer()
all_words_neg = sentim_analyzer.all_words([mark_negation(doc) for doc in training_docs])
示例11: SentimentAnalyzer
print "creating data set"
i = 0
s1 = ""
s2 = ""
tup = (s1, s2)
for line in f:
if i > 6718:
break
if i % 2 == 0:
s1 = line.split()
else:
s2 = line
tup = (s1, s2)
train.append(tup)
i += 1
print train
sentim_analyzer = SentimentAnalyzer()
all_words_neg = sentim_analyzer.all_words([mark_negation(doc) for doc in train])
print all_words_neg
unigram_feats = sentim_analyzer.unigram_word_feats(all_words_neg)
print unigram_feats
sentim_analyzer.add_feat_extractor(extract_unigram_feats, unigrams=unigram_feats)
training_set = sentim_analyzer.apply_features(train)
trainer = MaxentClassifier.train
classifier = sentim_analyzer.train(trainer, training_set)
f = open('maxent_trained_with_80_percent.pickle', 'wb')
pickle.dump(classifier, f)
f.close()
示例12: int
training_subjective = subjective[: int(0.8 * n)]
test_subjective = subjective[int(0.8 * n) : n]
training_objective = objective[: int(0.8 * n)]
test_objective = objective[int(0.8 * n) : n]
# Now aggregate the training and test sets
training = training_subjective + training_objective
test = test_subjective + test_objective
## Apply sentiment analysis to data to extract new "features"
# Initialize sentiment analyzer object
sentiment_analyzer = SentimentAnalyzer()
# Mark all negative words in training data, using existing list of negative words
all_negative_words = sentiment_analyzer.all_words([mark_negation(data) for data in training])
unigram_features = sentiment_analyzer.unigram_word_feats(all_negative_words, min_freq=4)
len(unigram_features)
sentiment_analyzer.add_feat_extractor(extract_unigram_feats, unigrams=unigram_features)
training_final = sentiment_analyzer.apply_features(training)
test_final = sentiment_analyzer.apply_features(test)
## Traing model and test
model = NaiveBayesClassifier.train
示例13: read_input
text = tokenizer.tokenize(line[5].decode("utf-8"))
text = [token for token in text if token != u'\ufffd']
test.append((text, sent))
return test, train
# Read in annotated data
NUM_TRAIN = 10000
NUM_TEST = 2500
test, train = read_input("train.csv",NUM_TRAIN,NUM_TEST)
sentiment_analyzer = SentimentAnalyzer()
#all_words = sentiment_analyzer.all_words([mark_negation(doc[0]) for doc in train])
all_words = sentiment_analyzer.all_words([doc[0] for doc in train])
unigrams = sentiment_analyzer.unigram_word_feats(all_words, min_freq=4)
# print unigrams
sentiment_analyzer.add_feat_extractor(extract_unigram_feats, unigrams=unigrams)
training_set=sentiment_analyzer.apply_features(train)
test_set=sentiment_analyzer.apply_features(test)
trainer = NaiveBayesClassifier.train
classifier = sentiment_analyzer.train(trainer, training_set)
save_file(sentiment_analyzer, "sentiment_classifier.pkl")
for key,value in sorted(sentiment_analyzer.evaluate(test_set).items()):
print("{0}: {1}".format(key,value))
示例14: return
row = line.split(',')
sentiment = row[1]
tweet = row[3].strip()
translator = str.maketrans({key: None for key in string.punctuation})
tweet = tweet.translate(translator)
tweet = tweet.split(' ')
tweet_lower = []
for word in tweet:
tweet_lower.append(word.lower())
return (tweet_lower, sentiment)
#call the function on each row in the dataset
train_data = train_data_raw.map(lambda line: get_row(line))
#create a SentimentAnalyzer object
sentim_analyzer = SentimentAnalyzer()
#get list of stopwords (with _NEG) to use as a filter
stopwords_all = []
for word in stopwords.words('english'):
stopwords_all.append(word)
stopwords_all.append(word + '_NEG')
#take 10,000 Tweets from this training dataset for this example and get all the words
#that are not stop words
train_data_sample = train_data.take(10000)
all_words_neg = sentim_analyzer.all_words([mark_negation(doc) for doc in train_data_sample])
all_words_neg_nostops = [x for x in all_words_neg if x not in stopwords_all]
#create unigram features and extract features
unigram_feats = sentim_analyzer.unigram_word_feats(all_words_neg_nostops, top_n=200)
示例15: len
len(subj_docs), len(obj_docs)
(100, 100)
subj_docs[0]
(['smart', 'and', 'alert', ',', 'thirteen', 'conversations', 'about', 'one',
'thing', 'is', 'a', 'small', 'gem', '.'], 'subj')
train_subj_docs = subj_docs[:80]
test_subj_docs = subj_docs[80:100]
train_obj_docs = obj_docs[:80]
test_obj_docs = obj_docs[80:100]
training_docs = train_subj_docs+train_obj_docs
testing_docs = test_subj_docs+test_obj_docs
sentim_analyzer = SentimentAnalyzer()
all_words_neg = sentim_analyzer.all_words([mark_negation(doc) for doc in training_docs])
unigram_feats = sentim_analyzer.unigram_word_feats(all_words_neg, min_freq=1)
len(unigram_feats)
sentim_analyzer.add_feat_extractor(extract_unigram_feats, unigrams=unigram_feats)
training_set = sentim_analyzer.apply_features(training_docs)
test_set = sentim_analyzer.apply_features(testing_docs)
trainer = NaiveBayesClassifier.train
classifier = sentim_analyzer.train(trainer, training_set)
for key,value in sorted(sentim_analyzer.evaluate(test_set).items()):
print('{0}: {1}'.format(key, value))
from nltk.sentiment.vader import SentimentIntensityAnalyzer
sentences = ["VADER is smart, handsome, and funny.", # positive sentence example