本文整理汇总了Python中nltk.corpus.PlaintextCorpusReader类的典型用法代码示例。如果您正苦于以下问题:Python PlaintextCorpusReader类的具体用法?Python PlaintextCorpusReader怎么用?Python PlaintextCorpusReader使用的例子?那么恭喜您, 这里精选的类代码示例或许可以为您提供帮助。
在下文中一共展示了PlaintextCorpusReader类的15个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: extract_related_terms
def extract_related_terms(self):
re = ReportEnviroments()
new_corpus_clusters_fileids_list = PlaintextCorpusReader(re.cluster_corpus_path, '.*')
raw_text_list = []
for i in range(len(new_corpus_clusters_fileids_list.fileids())):
raw_text_list.extend([[new_corpus_clusters_fileids_list.raw(fileids=new_corpus_clusters_fileids_list.fileids()[i])]])
return raw_text_list
示例2: get_lm_features
def get_lm_features(dataset,output_file):
# Import the corpus reader
corpus_root = '/home1/c/cis530/data-hw2/'+dataset
# Define the folder where the files are situated
files_dataset = PlaintextCorpusReader(corpus_root, '.*')
fin_model = BigramModel('Finance',corpus_root)
hel_model = BigramModel('Health',corpus_root)
res_model = BigramModel('Computers_and_the_Internet',corpus_root)
co_model = BigramModel('Research',corpus_root)
output = open('/home1/c/cis530/data-hw2/'+output_file,'w')
for fileid in files_dataset.fileids():
# Output the docid
output.write(dataset+'/'+fileid+' ')
# Output the topic_name
topic_name=fileid.split('/')[0]
output.write(topic_name+' ')
word_list = files_dataset.words(fileid)
finprob,finper = fin_model.get_prob_and_per(word_list)
hlprob,hlper = hel_model.get_prob_and_per(word_list)
resprob,resper = res_model.get_prob_and_per(word_list)
coprob,coper = co_model.get_prob_and_per(word_list)
output.write('finprob:'+str(round(finprob,1))+' ')
output.write('hlprob:'+str(round(hlprob,1))+' ')
output.write('resprob:'+str(round(resprob,1))+' ')
output.write('coprob:'+str(round(coprob,1))+' ')
output.write('finper:'+str(round(finper,1))+' ')
output.write('hlper:'+str(round(hlper,1))+' ')
output.write('resper:'+str(round(resper,1))+' ')
output.write('coper:'+str(round(coper,1))+' ')
output.write('\n')
output.close()
示例3: corpus_from_directory
def corpus_from_directory(path, filetype='.*'):
'''
Make a corpus of all files in a given directory. Can limit type by passing
the desired extension, proper format is, e.g., '.*\.txt'
'''
corpus_reader = PlaintextCorpusReader(path, filetype)
return nltk.Text( corpus_reader.words() )
示例4: plot_cfreq
def plot_cfreq(self,corpus,patt,n):
wordlists = PlaintextCorpusReader(corpus,patt)
fileids = wordlists.fileids()
for id in fileids:
words = wordlists.words(id)
fre = FreqDist(word.lower() for word in words if word.isalpha())
return fre.plot(n,cumulative=True)
示例5: similar
def similar (text, word):
if re.match ("^[a-zA-Z0-9_\(\),\.]+$", text) and re.match ("^[a-zA-Z0-9_]+$", word):
text = '%s.txt' % text
f = open(os.path.join(CORPUS_ROOT, text), 'r')
source = f.read()
f.close()
corpus = PlaintextCorpusReader(CORPUS_ROOT, [text])
n_text = nltk.text.Text(corpus.words(text))
context_index = nltk.text.ContextIndex(n_text.tokens, filter=lambda x:x.isalpha(), key=lambda s:s.lower())
word = word.lower()
wci = context_index._word_to_contexts
result = []
if word in wci.conditions():
contexts = set(wci[word])
fd = nltk.probability.FreqDist(w for w in wci.conditions() for c in wci[w] if c in contexts and not w == word)
words = nltk.util.tokenwrap(fd.keys()[:20])
for middle_word in words.split(' '):
for context in contexts:
if re.search ("/" + context[0] + "(\W|\s)+" + middle_word + "(\W|\s)+" + context[1] + "/i", source) != 'none':
print (context[0], middle_word, context[1])
result.append ({'word': word, 'context_left': context[0], 'context_right': context[1]})
return dumps ({'name': text, 'word': word, 'result': result})
示例6: extractWordsOnly
def extractWordsOnly(self, article):
templist = []
listtextstring = []
articlename = article + '.txt'
#corpus_root = '/home/jesal/onedump/'
wl = PlaintextCorpusReader(corpus_root, '.*')
allwords = wl.words(fileids = articlename)
exturllist = self.extractexternalURL(article)
textstring = wl.raw(articlename)
for item in exturllist:
textstring = textstring.replace(item,' ')
#templist = re.sub(r'[.!,;?]', ' ', textstring).split()
templist = nltk.word_tokenize(textstring)
listtemp = []
for i in templist:
j = re.sub('[^A-Za-z]+', '', i)
listtemp.append(str(j))
templistfinal = []
templistfinal= self.removeEmpty(listtemp)
return templistfinal
示例7: setData
def setData(domain):
# domain variable can take one of the following values
#
# "chicago_crime_data",
# "economics",
# "software_vulnerability",
# "cyber_threat",
# "articles",
# "msds"
corpus_root = getRoot(domain) # based on the selected domain corpus root will hold the relative address of the corpus
wordlists = PlaintextCorpusReader(corpus_root, '.*') # NLTK's laintextCorpusReader load text files in the root
words = wordlists.words() # and extract all the words in each file
my_stopwords = nltk.corpus.stopwords.words('english') # my_stopwords holds a list of non-relevant (stop) words in english
content = [w for w in words if w.lower() not in my_stopwords] # stop words are removed
content = [w for w in content if len(w) > 2] # words shorther than two(2) characters are removed
content = [w for w in content if not w.isdigit()] # digit only words (e.g. "10", "30", "450") are removed
result = {}
# a list of related words is created for each word in the content variable
for word in content:
result[word] = []
for sset in wn.synsets(word): # the first synonym of a set is selected, this can be expanded to the rest of the words in the set for more accuracy but at the cost of performace
for synset in sset.hyponyms(): # a set of hyponyms is added for the main synonym
result[word].append(synset.name[0:synset.name.find('.')])
return result,content # both the synonyms and the original word corpus is returned
示例8: extractParasInList
def extractParasInList(name):
corpuslocation ='/Users/anis/seniorProject/aligned Paragraphs/algebra'
reader = PlaintextCorpusReader(corpuslocation, '.*\.txt')
# This gives the list of paragraphs. every paragraph list contains ist of sentences
# So it is a list of lists. Bunch of sentenses as a list joins together to make #lists of pararagraph
pList = []
paragraphlist = reader.paras(name) #'simpleTuring.txt'
numpara = len(paragraphlist)
for sentlist in paragraphlist:
#print sentlist
numsent = len(sentlist)
#print type(sentlist),
#print numsent
paraAsAList = []
# this loops through all the sentence lists and make them one list'''
for i in range(numsent):
paraAsAList = paraAsAList + sentlist[i]
#print paraAsAList # this is the whole parapragph as one list
paraAsAString = ""
for word in paraAsAList:
paraAsAString = paraAsAString + word + str(" ")
#print paraAsAString
pList.append(paraAsAString)
#print len(pList)
return pList
示例9: fileids
def fileids(self, years='*'):
"""
Returns list all files or files exist in specific folder(s)
>>> len(hr.fileids())
3206
>>> len(hr.fileids(years=1996))
157
>>> len(hr.fileids(years=[1996,2007]))
246
>>> hr.fileids()[0]
'1996/HAM2-960622.xml'
"""
if type(years) is int:
years = [str(years)]
if years=='*':
wordlists = PlaintextCorpusReader(self.hamshahri_root, '.*\.xml')
fids = wordlists.fileids()
return fids
else:
fids = []
for year in years:
wordlists = PlaintextCorpusReader(self.hamshahri_root, str(year) + '/.*\.xml')
fids = fids + wordlists.fileids()
return fids
示例10: prepare_pos_features
def prepare_pos_features(Language_model_set, output_file):
corpus_root = '/home1/c/cis530/data-hw2/' + Language_model_set
texts = PlaintextCorpusReader(corpus_root, '.*')
text = texts.words()
tagged_text = nltk.pos_tag(text)
merged_tag_text = mergeTags(tagged_text)
lists = seperate_pos(merged_tag_text)
nouns_dist = FreqDist(lists[0])
top_nouns = nouns_dist.keys()[:200]
verbs_dist = FreqDist(lists[1])
top_verbs =verbs_dist.keys()[:200]
advs_dist = FreqDist(lists[2])
top_advs =advs_dist.keys()[:100]
prep_dist = FreqDist(lists[3])
top_preps =prep_dist.keys()[:100]
adjs_dist = FreqDist(lists[4])
top_adjs =adjs_dist.keys()[:200]
out = open(output_file, 'w')
for n in top_nouns:
out.write('NN'+ n + '\n')
for v in top_verbs:
out.write('VV'+ v + '\n')
for av in top_advs:
out.write('ADV'+ av + '\n')
for p in top_preps:
out.write('PREP'+ p + '\n')
for aj in top_adjs:
out.write('ADJ'+ aj + '\n')
示例11: get_coarse_level_features
def get_coarse_level_features(dataset, output_file):
# Import the corpus reader
corpus_root = '/home1/c/cis530/data-hw2/'+dataset
# Define the folder where the files are situated
files_dataset = PlaintextCorpusReader(corpus_root, '.*')
# Open the output_file
output = open('/home1/c/cis530/data-hw2/'+output_file,'w')
# Read the stopwlist
stop_list = open('/home1/c/cis530/data-hw2/'+'stopwlist.txt').read()
types_stop_list=stop_list.split()
for fileid in files_dataset.fileids():
# Output the docid
output.write(dataset+'/'+fileid+' ')
# Output the topic_name
topic_name=fileid.split('/')[0]
output.write(topic_name+' ')
# Output the num_tokens
tokens=files_dataset.words(fileid)
output.write('tok:'+str(len(tokens))+' ')
# Output the num_types
types=set(tokens)
output.write('typ:'+str(len(types))+' ')
# Output the num_contents
output.write('con:'+str(len([w for w in tokens if w not in types_stop_list]))+' ')
# Output the num_sents
sents = files_dataset.sents(fileid)
output.write('sen:'+str(len(sents))+' ')
# Output the avg_slen
avg_slen=round(float(len(tokens))/float(len(sents)),2)
output.write('len:'+str(avg_slen)+' ')
# Output the num_caps
output.write('cap:'+str(len([w for w in tokens if w[0]>='A' and w[0]<='Z'])))
output.write('\n')
output.close()
示例12: tokenisation
def tokenisation (path):
tokens = []
min_length = 3
for dirs in os.walk(path):
corpus_root = dirs[0] #parcour l'arborescence du chemin
if corpus_root != path:
textlist = PlaintextCorpusReader(corpus_root,'.*')
for files in textlist.fileids():
test= corpus_root + '/' + files
fs = open(test,'r')
texte=fs.readlines()
texte=str(texte)
words = map(lambda word: word.lower(), wordpunct_tokenize(texte))
j=0
while j<len(words):
if words[j] not in cachedStopWords:
tokens.append(words[j] )
j+=1
fs.close()
p = re.compile('[a-zA-Z]+')
tokens_filtered = filter(lambda token: p.match(token) and len(token)>= min_length, tokens)
# vocab = []
# for words in tokens_filtered:
# vocab.append(SnowballStemmer("english").stem(words))
# tokens_filtered_sans = set(vocab)
tokens_filtered_sans = set(tokens_filtered)
tokens_filtered_sans = list(tokens_filtered_sans)
return tokens_filtered_sans
示例13: tokenize_report_sents
def tokenize_report_sents(self, report_of_the_time):
re = ReportEnviroments()
new_corpus_reports_fileids_list = PlaintextCorpusReader(re.original_reports_corpus_path, '.*')
raw_text = new_corpus_reports_fileids_list.raw(report_of_the_time)
sentencas_raw = sent_tokenize(raw_text)
original_report_path = str(new_corpus_reports_fileids_list.abspath(report_of_the_time))
return sentencas_raw, original_report_path, report_of_the_time
示例14: main
def main():
# Corpus Location
#for training data
posTrainCorpus = 'C:/Users/Abhinav/Desktop/Course work/NLP/txt_sentoken/pos_train'
negTrainCorpus = 'C:/Users/Abhinav/Desktop/Course work/NLP/txt_sentoken/neg_train'
#for test data
posTestCorpus = 'C:/Users/Abhinav/Desktop/Course work/NLP/txt_sentoken/pos_test'
negTestCorpus = 'C:/Users/Abhinav/Desktop/Course work/NLP/txt_sentoken/neg_test'
# Create Plain Text Corpus for training data
posCorpus = PlaintextCorpusReader(posTrainCorpus, '.*')
negCorpus = PlaintextCorpusReader(negTrainCorpus, '.*')
# Create Plain Text Corpus for test data
posTstCorpus = PlaintextCorpusReader(posTestCorpus, '.*')
negTstCorpus = PlaintextCorpusReader(negTestCorpus, '.*')
#GetBigrams
posBigrams = nltk.bigrams(posCorpus.words())
negBigrams = nltk.bigrams(negCorpus.words())
#Get no. of words per corpus
posWordLen = len(posCorpus.words())
negWordLen = len(negCorpus.words())
# Creating object of Lang_Model_classifier
obj1 = Lang_Model_Classifier()
obj1.freq_dst(posCorpus, negCorpus)
#For negative test data
for filename in os.listdir(negTestCorpus):
wordSet = negTstCorpus.words(filename)
print '**Unigram**'
unigr = obj1.perp(wordSet)
print unigr
print '**Bigram**'
bigr = obj1.perpBi(nltk.bigrams(wordSet))
print bigr
#For positive test data
for filename in os.listdir(posTestCorpus):
wordSet2 = posTstCorpus.words(filename)
print '**Unigram**'
posunigr = obj1.perp(wordSet2)
print posunigr
print '**Bigram**'
posbigr = obj1.perpBi(nltk.bigrams(wordSet2))
print posbigr
示例15: Document
class Document(object):
"""
A container object for a set of chapters.
This allows us to keep track of document frequencies when computing them the
first time so we don't repeat computations for common words. It also handles
the PlaintextCorpusReader functions for us.
"""
def __init__(self, chapter_paths):
"""
Create a new Document.
chapter_paths - A list of the paths for chapters in the document.
"""
self.corpus = PlaintextCorpusReader("", chapter_paths)
self.chapter_lists = self._sanitize_chapters()
self.chapter_dists = [(FreqDist(chapter), chapter) for chapter in
self.chapter_lists]
self.words = {}
def get_chapters(self):
return self.chapter_lists
def average_chapter_frequency(self, word):
freqs = []
if word in self.words:
return self.words[word]
else:
for (dist, wordlist) in self.chapter_dists:
freqs.append(dist[word]/float(len(wordlist)))
# Store and return the average frequency
avg_frq = mean(freqs)
self.words[word] = avg_frq
return avg_frq
def _sanitize_chapters(self):
# Sanitize the wordlists and return them
lists = [self.corpus.words(file_id) for file_id in
self.corpus.fileids()]
new_lists = []
for word_list in lists:
# Convert everything to lowercase (e.g. so "the" and "The" match)
word_list = [word.lower() for word in word_list]
# Remove any punctuation
word_list = [re.sub('\p{P}','',word) for word in word_list]
# Remove stopwords, punctuation, and any empty word
stops = stopwords.words('english')
stops.append('')
stops.append('said')
word_list = [word for word in word_list if (word not in stops and
word.isalpha())]
new_lists.append(word_list)
return new_lists