本文整理匯總了Python中preprocessor.Preprocessor.clearTokenCache方法的典型用法代碼示例。如果您正苦於以下問題：Python Preprocessor.clearTokenCache方法的具體用法？Python Preprocessor.clearTokenCache怎麽用？Python Preprocessor.clearTokenCache使用的例子？那麽, 這裏精選的方法代碼示例或許可以為您提供幫助。您也可以進一步了解該方法所在類preprocessor.Preprocessor的用法示例。
在下文中一共展示了Preprocessor.clearTokenCache方法的1個代碼示例，這些例子默認根據受歡迎程度排序。您可以為喜歡或者感覺有用的代碼點讚，您的評價將有助於係統推薦出更棒的Python代碼示例。
示例1: analyze
# 需要導入模塊: from preprocessor import Preprocessor [as 別名]
# 或者: from preprocessor.Preprocessor import clearTokenCache [as 別名]
def analyze(snd_pipe, db_path, pp_cfg, parser_cfg, srcFiles, use_pipeline=False, analyzer_process=1, pp_process=1, parser_process=1):
db = DatabaseManager()
pp_list = [Preprocessor(**pp_cfg) for i in range(pp_process if use_pipeline else analyzer_process)]
parser_list = [Parser(**parser_cfg) for i in range(parser_process if use_pipeline else analyzer_process)]
numFiles = len(srcFiles)
use_pipeline = use_pipeline
t_0 = datetime.datetime.now()
projInfo = {}
projInfo['predefined'] = pp_list[0].preprocess_predef()
task_queue = Queue()
done_queue = Queue()
for i, srcFile in enumerate(srcFiles):
task_queue.put(srcFile)
for i in range(len(pp_list)):
task_queue.put('STOP')
if not use_pipeline:
analyzer_p_list = [Process(target=analyzer_worker, args=(pp, parser, task_queue, done_queue)) for pp, parser in zip(pp_list, parser_list)]
for analyzer_p in analyzer_p_list:
analyzer_p.start()
for i, srcFile in enumerate(srcFiles):
#print 'analyze: [%d/%d]' % (i,numFiles), srcFile
projInfo[srcFile] = done_queue.get()
snd_pipe.send((i, numFiles, srcFile))
if snd_pipe.poll():
for analyzer_p in analyzer_p_list:
analyzer_p.terminate()
for analyzer_p in analyzer_p_list:
analyzer_p.join()
Preprocessor.clearTokenCache()
snd_pipe.send('STOPPED')
print 'analyze: canceled'
return
for analyzer_p in analyzer_p_list:
analyzer_p.join()
else:
pp_queue = Queue()
pp_p_list = [Process(target=preprocessor_worker, args=(pp, task_queue, pp_queue)) for pp in pp_list]
for pp_p in pp_p_list:
pp_p.start()
parser_p_list = [Process(target=parser_worker, args=(parser, pp_queue, done_queue)) for parser in parser_list]
for parser_p in parser_p_list:
parser_p.start()
for i, srcFile in enumerate(srcFiles):
#print 'analyze: [%d/%d]' % (i,numFiles), srcFile
projInfo[srcFile] = done_queue.get()
snd_pipe.send((i, numFiles, srcFile))
if snd_pipe.poll():
for pp_p in pp_p_list:
pp_p.terminate()
for parser_p in parser_p_list:
parser_p.terminate()
for pp_p in pp_p_list:
pp_p.join()
for parser_p in parser_p_list:
parser_p.join()
Preprocessor.clearTokenCache()
snd_pipe.send('STOPPED')
print 'analyze: canceled'
return
for i in range(len(parser_p_list)):
pp_queue.put('STOP')
for pp_p in pp_p_list:
pp_p.join()
for parser_p in parser_p_list:
parser_p.join()
t_1 = datetime.datetime.now()
db.createDB(db_path)
db.addData(projInfo)
db.saveDB()
db.closeDB()
print 'analyze: done', t_1 - t_0
snd_pipe.send((numFiles, numFiles, 'Generating Database ... done'))