当前位置: 首页>>代码示例>>Python>>正文


Python PlaintextCorpusReader.fileids方法代码示例

本文整理汇总了Python中nltk.corpus.PlaintextCorpusReader.fileids方法的典型用法代码示例。如果您正苦于以下问题:Python PlaintextCorpusReader.fileids方法的具体用法?Python PlaintextCorpusReader.fileids怎么用?Python PlaintextCorpusReader.fileids使用的例子?那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在nltk.corpus.PlaintextCorpusReader的用法示例。


在下文中一共展示了PlaintextCorpusReader.fileids方法的15个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Python代码示例。

示例1: extract_related_terms

# 需要导入模块: from nltk.corpus import PlaintextCorpusReader [as 别名]
# 或者: from nltk.corpus.PlaintextCorpusReader import fileids [as 别名]
 def extract_related_terms(self):
     re = ReportEnviroments()
     new_corpus_clusters_fileids_list = PlaintextCorpusReader(re.cluster_corpus_path, '.*')
     raw_text_list = []
     for i in range(len(new_corpus_clusters_fileids_list.fileids())):
         raw_text_list.extend([[new_corpus_clusters_fileids_list.raw(fileids=new_corpus_clusters_fileids_list.fileids()[i])]])
     return raw_text_list
开发者ID:EduardoCarvalho,项目名称:nltkSegmenter,代码行数:9,代码来源:extractCluster.py

示例2: fileids

# 需要导入模块: from nltk.corpus import PlaintextCorpusReader [as 别名]
# 或者: from nltk.corpus.PlaintextCorpusReader import fileids [as 别名]
	def fileids(self, years='*'):
		"""
			Returns list all files or files exist in specific folder(s)
			
			>>> len(hr.fileids())
			3206
			>>> len(hr.fileids(years=1996))
			157
			>>> len(hr.fileids(years=[1996,2007]))
			246
			>>> hr.fileids()[0]
			'1996/HAM2-960622.xml'
		"""
		if type(years) is int:
			years = [str(years)]
		
		if years=='*':
			wordlists = PlaintextCorpusReader(self.hamshahri_root, '.*\.xml')
			fids = wordlists.fileids()
			return fids
		else:
			fids = []
			for year in years:
				wordlists = PlaintextCorpusReader(self.hamshahri_root, str(year) + '/.*\.xml')
				fids = fids + wordlists.fileids()
			return fids
开发者ID:alifars,项目名称:hazm,代码行数:28,代码来源:HamshahriReader.py

示例3: main

# 需要导入模块: from nltk.corpus import PlaintextCorpusReader [as 别名]
# 或者: from nltk.corpus.PlaintextCorpusReader import fileids [as 别名]
def main():
	current_directory = os.path.dirname(__file__)
	corpus_root = os.path.abspath(current_directory)
	wordlists = PlaintextCorpusReader(corpus_root, 'Islip13Rain/.*\.txt')
	wordlists.fileids()
	ClassEvent = nltk.Text(wordlists.words())
	CEWords = ["Long Island", "Weather Service", "flooding", "August", 
		"heavy rains", "Wednesday", "Suffolk County", "New York", "rainfall",
		"record"]

	# ClassEvent Statistics
	print "--------- CLASS EVENT STATISTICS -------------"
	print "ClassEvent non stopwords", non_stopword_fraction(ClassEvent)	
	print "ClassEvent WORD LENGTH DISTRIBUTIONS:"
	print_word_length_distributions(ClassEvent)
	print "ClassEvent PERCENTAGE OF WORD OCCURRENCES:"
	print_percentage_of_word_in_collection(ClassEvent, CEWords)
	
	ClassEventLettersPerWord = average_letters_per_word(ClassEvent)
	ClassEventWordsPerSent = len(wordlists.words()) / len(wordlists.sents())
	ClassEventARI = (4.71 * ClassEventLettersPerWord) + (0.5 * \
		ClassEventWordsPerSent) - 21.43
	
	print "Average number of letters per word", ClassEventLettersPerWord
	print "Average number of words per sentence:", ClassEventWordsPerSent
	print "Automated Readability Index:", ClassEventARI


	print 

	wordlists_event = PlaintextCorpusReader(corpus_root, "Texas_Wild_Fire/.*\.txt")
	wordlists_event.fileids()
	YourSmall = nltk.Text(wordlists_event.words())
	SmallEventWords = ["Fire", "Wildfire", "Water", "Damage", "Ground", "Burn", 
		"Town", "Heat", "Wind", "Speed", "Size", "City", "People", "Home",
		"Weather", "Debris", "Death", "Smoke", "State", "Ash"]
	

	# YourSmall statistics
	print "--------- YOUR SMALL STATISTICS --------------"
	print "Texas_Wild_Fire", non_stopword_fraction(YourSmall)
	print "YourSmall WORD LENGTH DISTRIBUTIONS:"
	print_word_length_distributions(YourSmall)
	print "YourSmall PERCENTAGE OF WORD OCCURRENCES:"
	print_percentage_of_word_in_collection(YourSmall, SmallEventWords)
	
	YourSmallLettersPerWord = average_letters_per_word(YourSmall)
	YourSmallWordsPerSent = len(wordlists_event.words()) / \
		len(wordlists_event.sents())
	YourSmallARI = (4.71 * YourSmallLettersPerWord) + (0.5 * \
		YourSmallWordsPerSent) - 21.43

	print "Average number of letters per word", YourSmallLettersPerWord
	print "Average number of words per sentence:", YourSmallWordsPerSent
	print "Automated Readability Index", YourSmallARI
开发者ID:jplahn,项目名称:NLP-Capstone,代码行数:57,代码来源:Statistics.py

示例4: carga

# 需要导入模块: from nltk.corpus import PlaintextCorpusReader [as 别名]
# 或者: from nltk.corpus.PlaintextCorpusReader import fileids [as 别名]
def carga():
    client = pymongo.MongoClient(MONGODB_URI)
    db = client.docs
    docs=db.SIMILITUD

    completo=[]
    newcorpus = PlaintextCorpusReader(corpus_root, '.*')
    result={}
    for fileid in newcorpus.fileids():
        for file2 in newcorpus.fileids():
            result= {"f1": fileid, "f2":file2, "value": compare_texts(newcorpus.raw(fileid), newcorpus.raw(file2))}
            docs.insert_one(result).inserted_id
开发者ID:jorpramo,项目名称:TAKESI_LOADER,代码行数:14,代码来源:prueba.py

示例5: loadCorpora

# 需要导入模块: from nltk.corpus import PlaintextCorpusReader [as 别名]
# 或者: from nltk.corpus.PlaintextCorpusReader import fileids [as 别名]
def loadCorpora():

    corpus_root = '/usr/share/dict'
    wordlists = PlaintextCorpusReader(corpus_root, '.*')
    wordlists.fileids()
    wordlists.words('connectives')

    corpus_root = r"C:\corpora\penntreebank\parsed\mrg\wsj"
    file_pattern = r".*/wsj_.*\.mrg" 
    ptb = BracketParseCorpusReader(corpus_root, file_pattern)
    ptb.fileids()
    len(ptb.sents())
    ptb.sents(fileids='20/wsj_2013.mrg')[19]
开发者ID:AkiraKane,项目名称:Python,代码行数:15,代码来源:c02_text_corpora.py

示例6: __init__

# 需要导入模块: from nltk.corpus import PlaintextCorpusReader [as 别名]
# 或者: from nltk.corpus.PlaintextCorpusReader import fileids [as 别名]
class Documents:
    def __init__(self, root):
        self.files = PlaintextCorpusReader(root, '.*\.txt')
        self.posting = {}
        self.idf = {}
        self.file_length = {}
        self.file_id_names = {}
        self.N = len(self.files.fileids())

    def process(self):
        for idx, file in enumerate(self.files.fileids()):
            print idx
            filename = file.strip('.txt')
            self.file_id_names[idx] = filename
            text = self.files.raw(file)
            words = process(text)
            if settings['phrase_query']:
                raw_words = raw_process(text)
            if words.values():
                self.file_length[idx] = normalization(words.values())
            for word, freq in words.iteritems():
                if self.idf.has_key(word):
                    self.idf[word] += 1
                else:
                    self.idf[word]  = 1

                if not self.posting.has_key(word):
                    self.posting[word] = {}
                if settings['phrase_query']:
                    self.posting[word][idx] = indices(raw_words, word)
                else:
                    self.posting[word][idx] = freq
        for word, idf in self.idf.iteritems():
            self.posting[word]['idf'] = idf

    def dump(self):
        posting_pickle = open('posting.pkl', 'wb')
        for term, value in self.posting.iteritems():
          self.posting[term] = str(value)
        pickle.dump(self.posting, posting_pickle, 2)
        posting_pickle.close()

        length_pickle = open('file_length.pkl', 'wb')
        pickle.dump(self.file_length, length_pickle, 2)
        length_pickle.close()

        file_ids_pickle = open('file_ids.pkl', 'wb')
        pickle.dump(self.file_id_names, file_ids_pickle, 2)
        file_ids_pickle.close()
开发者ID:jiexinhuang,项目名称:ir_engine,代码行数:51,代码来源:documents.py

示例7: represent_docs

# 需要导入模块: from nltk.corpus import PlaintextCorpusReader [as 别名]
# 或者: from nltk.corpus.PlaintextCorpusReader import fileids [as 别名]
def represent_docs(corpus,cat,dictio_classes,categories):
    
    docs_train = []
    for dirs in os.walk(corpus):
        corpus_root = dirs[0] #parcour l'arborescence du chemin
        
        if corpus_root != corpus:
            if os.path.basename(corpus_root) == cat:
                dictio = dictio_classes[cat]
                textlist = PlaintextCorpusReader(corpus_root,'.*')
                for files in textlist.fileids():
                    test= corpus_root + '/' + files
                    x = open(test,'r')
                    l=dictio.items()
                    l.sort(key=itemgetter(1),reverse=True)
                    l=l[:2000]
                    l=dict(l)               
                    for mot,fval in l.items():
                        val=fval
                        for ligne in x:
                            if ligne.find(mot)>0:
                                l[mot]=val
                            else:
                                l[mot]=0.0                     
                    x.close()
                    docs_train.append((l,'Yes'))
            else:
                if os.path.basename(corpus_root) in categories:
                    cat_else = os.path.basename(corpus_root)
                    dictio = dictio_classes[cat_else]
                    textlist = PlaintextCorpusReader(corpus_root,'.*')
                    for files in textlist.fileids():
                        test= corpus_root + '/' + files
                        x = open(test,'r')
                        l=dictio.items()
                        l.sort(key=itemgetter(1),reverse=True)
                        l=l[:2000]
                        l=dict(l)               
                        for mot,fval in l.items():
                            val_1=fval
                            for ligne in x:
                                if ligne.find(mot)>0:
                                    l[mot]=val_1
                                else:
                                    l[mot]=0.0                 
                        x.close()
                        docs_train.append((l,'No'))        
    return docs_train
开发者ID:priscileSua,项目名称:Pris,代码行数:50,代码来源:Yannis_ME.py

示例8: preprocTrain

# 需要导入模块: from nltk.corpus import PlaintextCorpusReader [as 别名]
# 或者: from nltk.corpus.PlaintextCorpusReader import fileids [as 别名]
def preprocTrain(corpus, tf_file, vocab_file):
    global MIN_FREQ
    stopwds = stopwords.words('english')

    TF = {} #gets the freq for each token
    filter_TF = {} #get the freq for each token having freq > minFreq
    feature_train = {} #final features for training class. Passed on to write ARFF files
    vocabulary = []
    ctDocs = {}
    totalDocs = 0
    minFreq = MIN_FREQ
    TrainingFiles = {}

    #loading our corpus
    corpus_root=corpus
    wordlists = PlaintextCorpusReader(corpus_root, '.*')
    ctDocs = len(wordlists.fileids()) #total no of files in each class
    totalDocs = ctDocs + totalDocs #total no of files
    TrainingFiles = wordlists.fileids() #contains files for each class

    sys.stderr.write("Reading corpus")
    for fileid in wordlists.fileids():
        sys.stderr.write(".")

        raw = wordlists.raw(fileid)
        tokens = nltk.word_tokenize(raw)
        text = nltk.Text(tokens)

        words = [w.lower() for w in text if w.isalnum() and w.lower() not in stopwds and len(w) > 3]
        vocab = set(words)
        words = nltk.Text(words)

        #calculate TF
        TF[fileid] = {fileid:fileid}
        filter_TF[fileid] = {fileid:fileid}
        for token in vocab:
            TF[fileid][token] = freq(token, words)

            if TF[fileid][token] > minFreq:  #min feature freq.
                vocabulary.append(token)
                filter_TF[fileid][token] = tf(TF[fileid][token],words)

    pickle.dump(filter_TF, open(tf_file, "wb"));
    sys.stderr.write("done\nCalculating TF*IDF scores")
    all_vocabulary = list(set(vocabulary))
    pickle.dump(all_vocabulary, open(vocab_file, "wb"));
    #featureIDF = idf(totalDocs,filter_TF,all_vocabulary)
    pprint(TF, stream=sys.stderr)
开发者ID:yadudoc,项目名称:swift-hadoop,代码行数:50,代码来源:calculate_tf_scores.py

示例9: save_my_count

# 需要导入模块: from nltk.corpus import PlaintextCorpusReader [as 别名]
# 或者: from nltk.corpus.PlaintextCorpusReader import fileids [as 别名]
 def save_my_count(self,corpus,patt,n,filename):
     wordlists = PlaintextCorpusReader(corpus,patt)
     fileids = wordlists.fileids()
     res = []
     for id in fileids:    
         leng = len(wordlists.words(id))
         wordc = len(set(wordlists.words(id)))
         wor = "=> corpus tokens: " + `leng` + "\n"
         dis = "=> corpus token types: " + `wordc` + "\n"
         ric = "=> ind lex richness: " + `leng / wordc` + "\n"
         res.append(dis)
         res.append(ric)
         res.append(wor)
         for word in sorted(set(wordlists.words(id))):
             freq = (wordlists.words(id)).count(word)
             f = "(" + word.lower() + "," + `round(100 * (freq / leng),1)` + ")\n"
             t = "(" + word.lower() + "," + `freq` + "/" + `leng` + ")"
             res.append(f)
             res.append(t)
     out = open("../data/"+filename,"w")
     try:
         for t in res[:n]:
             out.write(t + "\n")
     finally:
         out.close()
开发者ID:camilothorne,项目名称:nasslli2016,代码行数:27,代码来源:lexstatistics.py

示例10: get_lm_features

# 需要导入模块: from nltk.corpus import PlaintextCorpusReader [as 别名]
# 或者: from nltk.corpus.PlaintextCorpusReader import fileids [as 别名]
def get_lm_features(dataset,output_file):
        # Import the corpus reader
	corpus_root = '/home1/c/cis530/data-hw2/'+dataset
	# Define the folder where the files are situated
	files_dataset = PlaintextCorpusReader(corpus_root, '.*')	
        fin_model = BigramModel('Finance',corpus_root)
        hel_model = BigramModel('Health',corpus_root)
        res_model = BigramModel('Computers_and_the_Internet',corpus_root)
        co_model = BigramModel('Research',corpus_root)
        output = open('/home1/c/cis530/data-hw2/'+output_file,'w')
        for fileid in files_dataset.fileids():
		# Output the docid
		output.write(dataset+'/'+fileid+' ')
		# Output the topic_name
		topic_name=fileid.split('/')[0]
		output.write(topic_name+' ')		
		word_list = files_dataset.words(fileid)
		finprob,finper = fin_model.get_prob_and_per(word_list)		
		hlprob,hlper = hel_model.get_prob_and_per(word_list)	
		resprob,resper = res_model.get_prob_and_per(word_list)
		coprob,coper = co_model.get_prob_and_per(word_list)
		output.write('finprob:'+str(round(finprob,1))+' ')
		output.write('hlprob:'+str(round(hlprob,1))+' ')
		output.write('resprob:'+str(round(resprob,1))+' ')
		output.write('coprob:'+str(round(coprob,1))+' ')
		output.write('finper:'+str(round(finper,1))+' ')
		output.write('hlper:'+str(round(hlper,1))+' ')
		output.write('resper:'+str(round(resper,1))+' ')
		output.write('coper:'+str(round(coper,1))+' ')
		output.write('\n')
        output.close()
开发者ID:gabhi,项目名称:new-york-times-summarization,代码行数:33,代码来源:topic-classification.py

示例11: plot_cfreq

# 需要导入模块: from nltk.corpus import PlaintextCorpusReader [as 别名]
# 或者: from nltk.corpus.PlaintextCorpusReader import fileids [as 别名]
 def plot_cfreq(self,corpus,patt,n):
     wordlists = PlaintextCorpusReader(corpus,patt)
     fileids = wordlists.fileids()
     for id in fileids:
         words = wordlists.words(id)
         fre = FreqDist(word.lower() for word in words if word.isalpha()) 
     return fre.plot(n,cumulative=True)
开发者ID:camilothorne,项目名称:nasslli2016,代码行数:9,代码来源:lexstatistics.py

示例12: tokenisation

# 需要导入模块: from nltk.corpus import PlaintextCorpusReader [as 别名]
# 或者: from nltk.corpus.PlaintextCorpusReader import fileids [as 别名]
def tokenisation (path):
    tokens = []
    min_length = 3
    for dirs in os.walk(path):
        corpus_root = dirs[0] #parcour l'arborescence du chemin
        if corpus_root != path:
            textlist = PlaintextCorpusReader(corpus_root,'.*')
            for files in textlist.fileids():
                test= corpus_root + '/' + files
                fs = open(test,'r')
                texte=fs.readlines()
                texte=str(texte)
                words = map(lambda word: word.lower(), wordpunct_tokenize(texte))
                j=0
                while j<len(words):
                    if words[j] not in cachedStopWords:
                        tokens.append(words[j] )
                    j+=1
                fs.close() 
    p = re.compile('[a-zA-Z]+')
    tokens_filtered = filter(lambda token: p.match(token) and len(token)>= min_length, tokens)

#    vocab = []
#    for words in tokens_filtered:
#        vocab.append(SnowballStemmer("english").stem(words))
    
#    tokens_filtered_sans = set(vocab)
    tokens_filtered_sans = set(tokens_filtered)
    tokens_filtered_sans = list(tokens_filtered_sans)
    
    return tokens_filtered_sans
开发者ID:priscileSua,项目名称:Pris,代码行数:33,代码来源:Yannis_ME.py

示例13: get_coarse_level_features

# 需要导入模块: from nltk.corpus import PlaintextCorpusReader [as 别名]
# 或者: from nltk.corpus.PlaintextCorpusReader import fileids [as 别名]
def get_coarse_level_features(dataset, output_file):
	# Import the corpus reader
	corpus_root = '/home1/c/cis530/data-hw2/'+dataset
	# Define the folder where the files are situated
	files_dataset = PlaintextCorpusReader(corpus_root, '.*')
	# Open the output_file
	output = open('/home1/c/cis530/data-hw2/'+output_file,'w')
	# Read the stopwlist
	stop_list = open('/home1/c/cis530/data-hw2/'+'stopwlist.txt').read()
	types_stop_list=stop_list.split()
	for fileid in files_dataset.fileids():
		# Output the docid
		output.write(dataset+'/'+fileid+' ')
		# Output the topic_name
		topic_name=fileid.split('/')[0]	
		output.write(topic_name+' ')
		# Output the num_tokens	
		tokens=files_dataset.words(fileid)
		output.write('tok:'+str(len(tokens))+' ')
		# Output the num_types
		types=set(tokens)
		output.write('typ:'+str(len(types))+' ')
		# Output the num_contents
		output.write('con:'+str(len([w for w in tokens if w not in types_stop_list]))+' ')
		# Output the num_sents
		sents = files_dataset.sents(fileid)
		output.write('sen:'+str(len(sents))+' ')
		# Output the avg_slen
		avg_slen=round(float(len(tokens))/float(len(sents)),2)
		output.write('len:'+str(avg_slen)+' ')
		# Output the num_caps
		output.write('cap:'+str(len([w for w in tokens if w[0]>='A' and w[0]<='Z'])))
		output.write('\n')
	output.close()
开发者ID:gabhi,项目名称:new-york-times-summarization,代码行数:36,代码来源:topic-classification.py

示例14: Document

# 需要导入模块: from nltk.corpus import PlaintextCorpusReader [as 别名]
# 或者: from nltk.corpus.PlaintextCorpusReader import fileids [as 别名]
class Document(object):
    """
    A container object for a set of chapters.

    This allows us to keep track of document frequencies when computing them the
    first time so we don't repeat computations for common words. It also handles
    the PlaintextCorpusReader functions for us.
    """

    def __init__(self, chapter_paths):
        """
        Create a new Document.

        chapter_paths - A list of the paths for chapters in the document.
        """
        self.corpus = PlaintextCorpusReader("", chapter_paths)
        self.chapter_lists = self._sanitize_chapters()
        self.chapter_dists = [(FreqDist(chapter), chapter) for chapter in
                self.chapter_lists]
        self.words = {}

    def get_chapters(self):
        return self.chapter_lists

    def average_chapter_frequency(self, word):
        freqs = []
        if word in self.words:
            return self.words[word]
        else:
            for (dist, wordlist) in self.chapter_dists:
                freqs.append(dist[word]/float(len(wordlist)))

            # Store and return the average frequency
            avg_frq = mean(freqs)
            self.words[word] = avg_frq
            return avg_frq

    def _sanitize_chapters(self):
        # Sanitize the wordlists and return them
        lists = [self.corpus.words(file_id) for file_id in
                self.corpus.fileids()]

        new_lists = []

        for word_list in lists:
            # Convert everything to lowercase (e.g. so "the" and "The" match)
            word_list = [word.lower() for word in word_list]
            # Remove any punctuation
            word_list = [re.sub('\p{P}','',word) for word in word_list]
            # Remove stopwords, punctuation, and any empty word
            stops = stopwords.words('english')
            stops.append('')
            stops.append('said')
            word_list = [word for word in word_list if (word not in stops and
                word.isalpha())]

            new_lists.append(word_list)

        return new_lists
开发者ID:Dylnuge,项目名称:once-and-future-vis,代码行数:61,代码来源:document.py

示例15: get_sub_directories

# 需要导入模块: from nltk.corpus import PlaintextCorpusReader [as 别名]
# 或者: from nltk.corpus.PlaintextCorpusReader import fileids [as 别名]
def get_sub_directories(directory):
    files = PlaintextCorpusReader(directory, ".*")
    dirs = list()
    for f in files.fileids():
        if "/" in f:
            if (f[:f.index("/")] not in dirs):
                dirs.append(f[:f.index("/")])
    return dirs
开发者ID:closen39,项目名称:CIS-530-HW2,代码行数:10,代码来源:hw2_code_jmow_closen.py


注:本文中的nltk.corpus.PlaintextCorpusReader.fileids方法示例由纯净天空整理自Github/MSDocs等开源代码及文档管理平台,相关代码片段筛选自各路编程大神贡献的开源项目,源码版权归原作者所有,传播和使用请参考对应项目的License;未经允许,请勿转载。