当前位置: 首页>>代码示例>>Python>>正文


Python corpus.PlaintextCorpusReader类代码示例

本文整理汇总了Python中nltk.corpus.PlaintextCorpusReader的典型用法代码示例。如果您正苦于以下问题:Python PlaintextCorpusReader类的具体用法?Python PlaintextCorpusReader怎么用?Python PlaintextCorpusReader使用的例子?那么恭喜您, 这里精选的类代码示例或许可以为您提供帮助。


在下文中一共展示了PlaintextCorpusReader类的15个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Python代码示例。

示例1: extract_related_terms

 def extract_related_terms(self):
     re = ReportEnviroments()
     new_corpus_clusters_fileids_list = PlaintextCorpusReader(re.cluster_corpus_path, '.*')
     raw_text_list = []
     for i in range(len(new_corpus_clusters_fileids_list.fileids())):
         raw_text_list.extend([[new_corpus_clusters_fileids_list.raw(fileids=new_corpus_clusters_fileids_list.fileids()[i])]])
     return raw_text_list
开发者ID:EduardoCarvalho,项目名称:nltkSegmenter,代码行数:7,代码来源:extractCluster.py

示例2: get_lm_features

def get_lm_features(dataset,output_file):
        # Import the corpus reader
	corpus_root = '/home1/c/cis530/data-hw2/'+dataset
	# Define the folder where the files are situated
	files_dataset = PlaintextCorpusReader(corpus_root, '.*')	
        fin_model = BigramModel('Finance',corpus_root)
        hel_model = BigramModel('Health',corpus_root)
        res_model = BigramModel('Computers_and_the_Internet',corpus_root)
        co_model = BigramModel('Research',corpus_root)
        output = open('/home1/c/cis530/data-hw2/'+output_file,'w')
        for fileid in files_dataset.fileids():
		# Output the docid
		output.write(dataset+'/'+fileid+' ')
		# Output the topic_name
		topic_name=fileid.split('/')[0]
		output.write(topic_name+' ')		
		word_list = files_dataset.words(fileid)
		finprob,finper = fin_model.get_prob_and_per(word_list)		
		hlprob,hlper = hel_model.get_prob_and_per(word_list)	
		resprob,resper = res_model.get_prob_and_per(word_list)
		coprob,coper = co_model.get_prob_and_per(word_list)
		output.write('finprob:'+str(round(finprob,1))+' ')
		output.write('hlprob:'+str(round(hlprob,1))+' ')
		output.write('resprob:'+str(round(resprob,1))+' ')
		output.write('coprob:'+str(round(coprob,1))+' ')
		output.write('finper:'+str(round(finper,1))+' ')
		output.write('hlper:'+str(round(hlper,1))+' ')
		output.write('resper:'+str(round(resper,1))+' ')
		output.write('coper:'+str(round(coper,1))+' ')
		output.write('\n')
        output.close()
开发者ID:gabhi,项目名称:new-york-times-summarization,代码行数:31,代码来源:topic-classification.py

示例3: corpus_from_directory

def corpus_from_directory(path, filetype='.*'):
	'''
	Make a corpus of all files in a given directory. Can limit type by passing
	the desired extension, proper format is, e.g., '.*\.txt'
	'''
	corpus_reader = PlaintextCorpusReader(path, filetype)
	return nltk.Text( corpus_reader.words() )
开发者ID:campustimes,项目名称:pnlp-final-project,代码行数:7,代码来源:corpustools.py

示例4: plot_cfreq

 def plot_cfreq(self,corpus,patt,n):
     wordlists = PlaintextCorpusReader(corpus,patt)
     fileids = wordlists.fileids()
     for id in fileids:
         words = wordlists.words(id)
         fre = FreqDist(word.lower() for word in words if word.isalpha()) 
     return fre.plot(n,cumulative=True)
开发者ID:camilothorne,项目名称:nasslli2016,代码行数:7,代码来源:lexstatistics.py

示例5: similar

def similar (text, word):
    if re.match ("^[a-zA-Z0-9_\(\),\.]+$", text) and re.match ("^[a-zA-Z0-9_]+$", word):
        text = '%s.txt' % text
        
        f = open(os.path.join(CORPUS_ROOT, text), 'r')
        source = f.read()
        f.close()
        
        corpus = PlaintextCorpusReader(CORPUS_ROOT, [text])
        n_text = nltk.text.Text(corpus.words(text))
        context_index = nltk.text.ContextIndex(n_text.tokens, filter=lambda x:x.isalpha(), key=lambda s:s.lower())
        word = word.lower()
        wci = context_index._word_to_contexts
        result = []
        
        if word in wci.conditions():
            contexts = set(wci[word])
            fd = nltk.probability.FreqDist(w for w in wci.conditions() for c in wci[w] if c in contexts and not w == word)
            words = nltk.util.tokenwrap(fd.keys()[:20])
            
            for middle_word in words.split(' '):
                for context in contexts:
                    if re.search ("/" + context[0] + "(\W|\s)+" + middle_word + "(\W|\s)+" + context[1] + "/i", source) != 'none':
                        print (context[0], middle_word, context[1])
                        result.append ({'word': word, 'context_left': context[0], 'context_right': context[1]})
            
        return dumps ({'name': text, 'word': word, 'result': result})    
开发者ID:osp,项目名称:osp.work.vj12,代码行数:27,代码来源:vj12.py

示例6: extractWordsOnly

    def extractWordsOnly(self, article):
        templist = []
        listtextstring = []
        articlename = article + '.txt'
        #corpus_root = '/home/jesal/onedump/'
        wl = PlaintextCorpusReader(corpus_root, '.*')
        allwords = wl.words(fileids = articlename)
        exturllist = self.extractexternalURL(article)
        textstring = wl.raw(articlename)
        for item in exturllist:
            textstring = textstring.replace(item,' ')
    

        
        #templist = re.sub(r'[.!,;?]', ' ', textstring).split()
        templist = nltk.word_tokenize(textstring)
        listtemp = []
        for i in templist:
        	j = re.sub('[^A-Za-z]+', '', i)
        	listtemp.append(str(j))
		    
		    
		    
		    
        templistfinal = []
        templistfinal= self.removeEmpty(listtemp)
        return templistfinal
开发者ID:Wiki-G,项目名称:wikiG-app,代码行数:27,代码来源:testImplemented.py

示例7: setData

def setData(domain):
	
	# domain variable can take one of the following values
	#
	# "chicago_crime_data",
	# "economics",
	# "software_vulnerability",
	# "cyber_threat",
	# "articles",
	# "msds"

	
	corpus_root = getRoot(domain)					# based on the selected domain corpus root will hold the relative address of the corpus
	wordlists = PlaintextCorpusReader(corpus_root, '.*')		# NLTK's laintextCorpusReader load text files in the root
	words = wordlists.words()					# and extract all the words in each file 

	my_stopwords = nltk.corpus.stopwords.words('english')		# my_stopwords holds a list of non-relevant (stop) words in english
	content = [w for w in words if w.lower() not in my_stopwords]	# stop words are removed
	content = [w for w in content if len(w) > 2]			# words shorther than two(2) characters are removed
	content = [w for w in content if not w.isdigit()]		# digit only words (e.g. "10", "30", "450") are removed

	result = {}							
	
	# a list of related words is created for each word in the content variable
	
	for word in content:						
		result[word] = []
		for sset in wn.synsets(word):				# the first synonym of a set is selected, this can be expanded to the rest of the words in the set for more accuracy but at the cost of performace
			for synset in sset.hyponyms():			# a set of hyponyms is added for the main synonym
				result[word].append(synset.name[0:synset.name.find('.')])

	return result,content # both the synonyms and the original word corpus is returned 
开发者ID:bpolania,项目名称:GlobalHack,代码行数:32,代码来源:ontology.py

示例8: extractParasInList

def extractParasInList(name):
    corpuslocation ='/Users/anis/seniorProject/aligned Paragraphs/algebra'
    reader = PlaintextCorpusReader(corpuslocation, '.*\.txt')
    # This gives the list of paragraphs. every paragraph list contains ist of sentences
    # So it is a list of lists. Bunch of sentenses as a list joins together to make  		#lists of pararagraph
    pList = []
    paragraphlist =  reader.paras(name) #'simpleTuring.txt'
    numpara = len(paragraphlist)
    for sentlist in paragraphlist:
        #print sentlist
        numsent = len(sentlist)
        #print type(sentlist),
        #print numsent
     	paraAsAList = []
     	# this loops through all the sentence lists and make them one list'''
        for i in range(numsent):
        		paraAsAList = paraAsAList + sentlist[i]	
        #print paraAsAList # this is the whole parapragph as one list
     	paraAsAString = ""
     	for word in paraAsAList:
        		paraAsAString = paraAsAString + word + str(" ")
        #print paraAsAString
        pList.append(paraAsAString)
        #print len(pList)
    return pList
开发者ID:azaman13,项目名称:senior-project,代码行数:25,代码来源:tf_idfBak.py

示例9: fileids

	def fileids(self, years='*'):
		"""
			Returns list all files or files exist in specific folder(s)
			
			>>> len(hr.fileids())
			3206
			>>> len(hr.fileids(years=1996))
			157
			>>> len(hr.fileids(years=[1996,2007]))
			246
			>>> hr.fileids()[0]
			'1996/HAM2-960622.xml'
		"""
		if type(years) is int:
			years = [str(years)]
		
		if years=='*':
			wordlists = PlaintextCorpusReader(self.hamshahri_root, '.*\.xml')
			fids = wordlists.fileids()
			return fids
		else:
			fids = []
			for year in years:
				wordlists = PlaintextCorpusReader(self.hamshahri_root, str(year) + '/.*\.xml')
				fids = fids + wordlists.fileids()
			return fids
开发者ID:alifars,项目名称:hazm,代码行数:26,代码来源:HamshahriReader.py

示例10: prepare_pos_features

def prepare_pos_features(Language_model_set, output_file):
    corpus_root = '/home1/c/cis530/data-hw2/' + Language_model_set
    texts = PlaintextCorpusReader(corpus_root, '.*')
    text = texts.words()
    tagged_text = nltk.pos_tag(text)
    merged_tag_text = mergeTags(tagged_text)
    lists = seperate_pos(merged_tag_text)
    nouns_dist = FreqDist(lists[0])
    top_nouns = nouns_dist.keys()[:200]
    verbs_dist = FreqDist(lists[1])
    top_verbs =verbs_dist.keys()[:200]
    advs_dist = FreqDist(lists[2])
    top_advs =advs_dist.keys()[:100]
    prep_dist = FreqDist(lists[3])
    top_preps =prep_dist.keys()[:100]
    adjs_dist = FreqDist(lists[4])
    top_adjs =adjs_dist.keys()[:200]


    out = open(output_file, 'w')

    for n in top_nouns:
        out.write('NN'+ n + '\n')
    for v in top_verbs:
        out.write('VV'+ v + '\n')
    for av in top_advs:
        out.write('ADV'+ av + '\n')
    for p in top_preps:
        out.write('PREP'+ p + '\n')
    for aj in top_adjs:
        out.write('ADJ'+ aj + '\n')
开发者ID:madhuraraju,项目名称:NLP_Class_Code_Samples,代码行数:31,代码来源:CL_Two_Code_rmadhura.py

示例11: get_coarse_level_features

def get_coarse_level_features(dataset, output_file):
	# Import the corpus reader
	corpus_root = '/home1/c/cis530/data-hw2/'+dataset
	# Define the folder where the files are situated
	files_dataset = PlaintextCorpusReader(corpus_root, '.*')
	# Open the output_file
	output = open('/home1/c/cis530/data-hw2/'+output_file,'w')
	# Read the stopwlist
	stop_list = open('/home1/c/cis530/data-hw2/'+'stopwlist.txt').read()
	types_stop_list=stop_list.split()
	for fileid in files_dataset.fileids():
		# Output the docid
		output.write(dataset+'/'+fileid+' ')
		# Output the topic_name
		topic_name=fileid.split('/')[0]	
		output.write(topic_name+' ')
		# Output the num_tokens	
		tokens=files_dataset.words(fileid)
		output.write('tok:'+str(len(tokens))+' ')
		# Output the num_types
		types=set(tokens)
		output.write('typ:'+str(len(types))+' ')
		# Output the num_contents
		output.write('con:'+str(len([w for w in tokens if w not in types_stop_list]))+' ')
		# Output the num_sents
		sents = files_dataset.sents(fileid)
		output.write('sen:'+str(len(sents))+' ')
		# Output the avg_slen
		avg_slen=round(float(len(tokens))/float(len(sents)),2)
		output.write('len:'+str(avg_slen)+' ')
		# Output the num_caps
		output.write('cap:'+str(len([w for w in tokens if w[0]>='A' and w[0]<='Z'])))
		output.write('\n')
	output.close()
开发者ID:gabhi,项目名称:new-york-times-summarization,代码行数:34,代码来源:topic-classification.py

示例12: tokenisation

def tokenisation (path):
    tokens = []
    min_length = 3
    for dirs in os.walk(path):
        corpus_root = dirs[0] #parcour l'arborescence du chemin
        if corpus_root != path:
            textlist = PlaintextCorpusReader(corpus_root,'.*')
            for files in textlist.fileids():
                test= corpus_root + '/' + files
                fs = open(test,'r')
                texte=fs.readlines()
                texte=str(texte)
                words = map(lambda word: word.lower(), wordpunct_tokenize(texte))
                j=0
                while j<len(words):
                    if words[j] not in cachedStopWords:
                        tokens.append(words[j] )
                    j+=1
                fs.close() 
    p = re.compile('[a-zA-Z]+')
    tokens_filtered = filter(lambda token: p.match(token) and len(token)>= min_length, tokens)

#    vocab = []
#    for words in tokens_filtered:
#        vocab.append(SnowballStemmer("english").stem(words))
    
#    tokens_filtered_sans = set(vocab)
    tokens_filtered_sans = set(tokens_filtered)
    tokens_filtered_sans = list(tokens_filtered_sans)
    
    return tokens_filtered_sans
开发者ID:priscileSua,项目名称:Pris,代码行数:31,代码来源:Yannis_ME.py

示例13: tokenize_report_sents

 def tokenize_report_sents(self, report_of_the_time):
     re = ReportEnviroments()
     new_corpus_reports_fileids_list = PlaintextCorpusReader(re.original_reports_corpus_path, '.*')
     raw_text = new_corpus_reports_fileids_list.raw(report_of_the_time)
     sentencas_raw = sent_tokenize(raw_text)
     original_report_path = str(new_corpus_reports_fileids_list.abspath(report_of_the_time))
     return sentencas_raw, original_report_path, report_of_the_time
开发者ID:EduardoCarvalho,项目名称:nltkSegmenter,代码行数:7,代码来源:reportSegmenter.py

示例14: main

def main():
    
    # Corpus Location
    #for training data
    posTrainCorpus = 'C:/Users/Abhinav/Desktop/Course work/NLP/txt_sentoken/pos_train'
    negTrainCorpus = 'C:/Users/Abhinav/Desktop/Course work/NLP/txt_sentoken/neg_train'

    #for test data
    posTestCorpus = 'C:/Users/Abhinav/Desktop/Course work/NLP/txt_sentoken/pos_test'
    negTestCorpus = 'C:/Users/Abhinav/Desktop/Course work/NLP/txt_sentoken/neg_test'

    # Create Plain Text Corpus for training data
    posCorpus = PlaintextCorpusReader(posTrainCorpus, '.*')
    negCorpus = PlaintextCorpusReader(negTrainCorpus, '.*')


    # Create Plain Text Corpus for test data
    posTstCorpus = PlaintextCorpusReader(posTestCorpus, '.*')
    negTstCorpus = PlaintextCorpusReader(negTestCorpus, '.*')
    
    #GetBigrams
    posBigrams = nltk.bigrams(posCorpus.words())
    negBigrams = nltk.bigrams(negCorpus.words())

    #Get no. of words per corpus
    posWordLen = len(posCorpus.words())
    negWordLen = len(negCorpus.words())
    
    # Creating object of Lang_Model_classifier
    obj1 = Lang_Model_Classifier()
    obj1.freq_dst(posCorpus, negCorpus)
    
    #For negative test data
    for filename in os.listdir(negTestCorpus):
        wordSet =  negTstCorpus.words(filename)
    
        print '**Unigram**'
        unigr = obj1.perp(wordSet)
    
        print unigr
    
        print '**Bigram**'
        bigr = obj1.perpBi(nltk.bigrams(wordSet))
    
        print bigr
        
    #For positive test data    
    for filename in os.listdir(posTestCorpus):
        wordSet2 =  posTstCorpus.words(filename)
    
        print '**Unigram**'
        posunigr = obj1.perp(wordSet2)
    
        print posunigr
    
        print '**Bigram**'
        posbigr = obj1.perpBi(nltk.bigrams(wordSet2))
    
        print posbigr
开发者ID:abhinavmishra590,项目名称:Sentiment-Based-Document-Classification,代码行数:59,代码来源:Lang_Model.py

示例15: Document

class Document(object):
    """
    A container object for a set of chapters.

    This allows us to keep track of document frequencies when computing them the
    first time so we don't repeat computations for common words. It also handles
    the PlaintextCorpusReader functions for us.
    """

    def __init__(self, chapter_paths):
        """
        Create a new Document.

        chapter_paths - A list of the paths for chapters in the document.
        """
        self.corpus = PlaintextCorpusReader("", chapter_paths)
        self.chapter_lists = self._sanitize_chapters()
        self.chapter_dists = [(FreqDist(chapter), chapter) for chapter in
                self.chapter_lists]
        self.words = {}

    def get_chapters(self):
        return self.chapter_lists

    def average_chapter_frequency(self, word):
        freqs = []
        if word in self.words:
            return self.words[word]
        else:
            for (dist, wordlist) in self.chapter_dists:
                freqs.append(dist[word]/float(len(wordlist)))

            # Store and return the average frequency
            avg_frq = mean(freqs)
            self.words[word] = avg_frq
            return avg_frq

    def _sanitize_chapters(self):
        # Sanitize the wordlists and return them
        lists = [self.corpus.words(file_id) for file_id in
                self.corpus.fileids()]

        new_lists = []

        for word_list in lists:
            # Convert everything to lowercase (e.g. so "the" and "The" match)
            word_list = [word.lower() for word in word_list]
            # Remove any punctuation
            word_list = [re.sub('\p{P}','',word) for word in word_list]
            # Remove stopwords, punctuation, and any empty word
            stops = stopwords.words('english')
            stops.append('')
            stops.append('said')
            word_list = [word for word in word_list if (word not in stops and
                word.isalpha())]

            new_lists.append(word_list)

        return new_lists
开发者ID:Dylnuge,项目名称:once-and-future-vis,代码行数:59,代码来源:document.py


注:本文中的nltk.corpus.PlaintextCorpusReader类示例由纯净天空整理自Github/MSDocs等开源代码及文档管理平台,相关代码片段筛选自各路编程大神贡献的开源项目,源码版权归原作者所有,传播和使用请参考对应项目的License;未经允许,请勿转载。