本文整理汇总了Python中multiprocessing.Manager.file方法的典型用法代码示例。如果您正苦于以下问题：Python Manager.file方法的具体用法？Python Manager.file怎么用？Python Manager.file使用的例子？那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类multiprocessing.Manager的用法示例。
在下文中一共展示了Manager.file方法的2个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: process
# 需要导入模块: from multiprocessing import Manager [as 别名]
# 或者: from multiprocessing.Manager import file [as 别名]
def process(infile):
# prepare and load file
print '\nStarted job', infile
data_file = load(infile)
manager = Manager()
manager.file = data_file
# Build a pool of num_cpus processes
pool = Pool(processes=int(num_cpus), initializer=init,
initargs=(data_file, ))
# Partition job into appropriate slices, prepare argument tuple for Map
offset = abs(len(data_file) / num_cpus)
print 'partitioned job to', num_cpus, 'slices of', humansize(offset)
partitioned_jobs = list(chunkjobs(infile, data_file, offset))
data_file.close()
# Generate count tuples for matched words from dictionary
print 'Mapping job', infile
single_count_tuples = pool.map(Map, partitioned_jobs)
# Organize the count tuples; lists of tuples by token key
token_to_tuples = Partition(single_count_tuples)
# Collapse the lists of tuples into total term frequencies
print 'Reducing job', infile
term_frequencies = pool.map(Reduce, token_to_tuples.items())
# Sort the term frequencies in nonincreasing order
term_frequencies.sort(tuple_sort)
# dump dictionary from count, we want this sorted
dumpdictionary(term_frequencies)
# Output
print 'top %d tokens by frequency' % int(args.top)
for (index, pair) in enumerate(term_frequencies[:int(args.top)]):
print index + 1, ':', pair[0], ':', pair[1]
output = open('term_frequencies_' + os.path.splitext(infile)[0]
+ '.txt', 'wt')
for pair in term_frequencies:
output.write(str(pair[0]) + ': ' + str(pair[1]) + '\n')
for n in percentages(pair):
output.write(' %d: %3.2f%% ' % (n[0], n[1]))
output.write('\n')
output.close()示例2: process
# 需要导入模块: from multiprocessing import Manager [as 别名]
# 或者: from multiprocessing.Manager import file [as 别名]
def process(infile):
# prepare and load file
print '\nStarted job', infile
data_file = load(infile)
# get either fslice or num_cpus
global num_cpus
num_cpus = getcpus(data_file)
manager = Manager()
manager.file = data_file
# Partition job into appropriate slices, prepare argument tuple for Map
print "Parsing job to sentences"
offset = len(data_file) / num_cpus
print 'partitioned',humansize(len(data_file)),'job to', num_cpus,\
'slices of', humansize(offset)
# Generate count tuples for matched words from dictionary
print 'Preparing job', infile
# Build a pool of num_cpus processes
pool = Pool(processes=int(num_cpus), initializer=init, initargs=(data_file,))
# Map the job
single_count_tuples = pool.map(Map,((i, infile) for i in xrange(num_cpus)))
data_file.close()
# Organize the count tuples; lists of tuples by token key
token_to_tuples = Partition(single_count_tuples)
# Collapse the lists of tuples into total term frequencies
print 'Reducing job', infile
term_frequencies = pool.map(Reduce, token_to_tuples.items())
# Sort the term frequencies in nonincreasing order
term_frequencies.sort(tuple_sort)
# dump dictionary from count, we want this sorted
dumpdictionary(term_frequencies)
# Output top term frequencies to console
print 'top %d tokens by frequency' % int(args.top)
for (index, pair) in enumerate(term_frequencies[:int(args.top)]):
print '%1d %2s %3d' % (index + 1, pair[0] ,pair[1])
# Dump human-readable statistics
dumpstats(term_frequencies)