Python threadpool.ThreadPool类代码示例

    def run(self, env, json_writer):
        '''
        Schedule all tests in profile for execution.

        See ``Test.schedule`` and ``Test.run``.
        '''

        self.prepare_test_list(env)

        # If using concurrency, add all the concurrent tests to the pool and
        # execute that pool
        if env.concurrent:
            pool = ThreadPool(multiprocessing.cpu_count())
            for (path, test) in self.test_list.items():
                if test.runConcurrent:
                    pool.add(test.execute, (env, path, json_writer))
            pool.join()

        # Run any remaining tests serially from a single thread pool after the
        # concurrent tests have finished
        pool = ThreadPool(1)
        for (path, test) in self.test_list.items():
            if not env.concurrent or not test.runConcurrent:
                pool.add(test.execute, (env, path, json_writer))
        pool.join()

 def testImageDownload(self):
     logging.debug('Start at %s', datetime.now())
     url = 'http://f1.163.com'
     # url = 'http://news.sina.com.cn/photo/'
     work_request = WorkRequest(spider, url)
     pool = ThreadPool(10, work_request)
     pool.poll()

def convertFlv2Mp4underDir(path):
    if not os.path.isdir(path):
        if os.path.exists(path):
            print "  Path:["+ path+ "] is not a directory, exit!\n"
            return
        else:
            os.makedirs(path)
    
    pool = ThreadPool(6)
    
    MP4_CMD = '''D:\\Program\\tools\\ffmpeg.exe -i "%s" -vcodec mpeg4 -b 1200kb -mbd 2 -aic 2 -cmp 2 -subcmp 2 -acodec libfaac -ac 2 -ab 128000 -y "%s"'''
    MP3_CMD = '''D:\\Program\\tools\\ffmpeg.exe -i "%s" -vn -ar 44100 -ac 2 -f mp3 "%s"'''
    for file_name in os.listdir(path):
        flv_path = path+'\\'+file_name
        if os.path.isfile(flv_path):
            mp4_file_name = file_name[:file_name.rfind('.')]+'.mp4'
            mp4_save_path = path+'\\mp4\\'+mp4_file_name
            if os.path.exists(mp4_save_path):
                print "  File:[" + mp4_save_path+ "] already exists, pass.\n"
            else:
                cmd = MP4_CMD%(flv_path, mp4_save_path)
                #print cmd
                #pool.queueTask(run_cmd, (cmd))
            
            mp3_file_name = file_name[:file_name.rfind('.')]+'.mp3'
            mp3_save_path = path+'\\mp3\\'+mp3_file_name
            if os.path.exists(mp3_save_path):
                print "  File:[" + mp3_save_path+ "] already exists, pass.\n"
            else:
                cmd = MP3_CMD%(flv_path, mp3_save_path)
                print cmd
                pool.queueTask(run_cmd, (cmd))
            
    pool.joinAll()

def downloadAllPagesVideos(url):
    global proxy, host, thread_count
    print url
    content = getContent(url, None, proxy)
    
    all_page_content = ''
    matched_groups = re.findall('''<a href="(.*?)" title='第\d+页' charset=".*?">\d+</a>''', content)
    for matched in matched_groups:
        page_url = 'http://so.youku.com'+matched.strip()
        all_page_content += getContent(page_url, None, proxy)
    
    
    pool = ThreadPool(thread_count)

    video_url_set = set()
    matched_groups = re.findall('''<a href="(http\://v\.youku\.com/v_show/id_.*?=\.html)"''', all_page_content)
    for matched in matched_groups:
        #print matched.strip()
        video_url = matched.strip()
        video_url_set.add(video_url)

    for video_url in video_url_set:
        print video_url
        log(video_url)
        pool.queueTask(downloadVideo, (video_url))

    pool.joinAll()

class Spider:
	def __init__(self, depth=2):
		self.threadPool = ThreadPool(10)
		self.depth = depth

	def start(self, currentLevel, url):
		self.threadPool.initPool()
		self.threadPool.putTask(self.crawlPage, \
							currentLevel = currentLevel, \
							url = url)

	def crawlPage(self, args):
		print 'crawlPage', args
		currentLevel = args['currentLevel']
		url = args['url']
		req = urllib2.Request(url=url, headers=header)
		try:
			resp = urllib2.urlopen(req, timeout=10)
		except urllib2.HTTPError as e:
			# XXX The except HTTPError must come first
			# otherwise except URLError will also catch an HTTPError
			pass
		except urllib2.URLError as e:
			pass
		except Exception, e:
			print e
		else:

def convertWMA2MP3underDir(path):
    if not os.path.isdir(path):
        if existFile(path):
            print "  Path:["+ path+ "] is not a directory, exit!\n"
            return
        else:
            os.makedirs(path)
    
    pool = ThreadPool(6)
    
    MP3_CMD = '''ffmpeg.exe -i "%s" -f mp3 "%s"'''
    DEL_CMD = '''del %s'''
    for file_name in os.listdir(path):
        wma_path = path+'\\'+file_name
        if os.path.isfile(wma_path) and wma_path.lower().endswith('.wma'):
            mp3_file_name = file_name[:file_name.rfind('.')]+'.mp3'
            mp3_save_path = path+'\\'+mp3_file_name
            if os.path.exists(mp3_save_path):
                print "  File:[" + mp3_save_path+ "] already exists, pass.\n"
            else:
                cmd1 = MP3_CMD%(wma_path, mp3_save_path)
                #cmd2 = DEL_CMD%(wma_path)
                print cmd1
                pool.queueTask(run_cmd, (cmd1))
        
    pool.joinAll()

def search(song, n, processes=config.search_processes, returnGen=False):
	'''
	Function searches song and returns n valid .mp3 links.
	@param song: Search string.
	@param n: Number of songs.
	@param processes: Number of processes to launch in the subprocessing pool.
	@param returnGen: If true, a generator of the links will be returned,
						and not the calculated list itself.
	'''
	sources_list = [x for x in config.search_sources_const if config.search_sources[x]]
	log.debug("Using sources: %s" % sources_list)
	
	# IMPROVE: better handeling of slicing.
	pool = ThreadPool(max_threads=min(processes, len(sources_list)), catch_returns=True, logger=log)
	args_list = []
	for source in sources_list:
		args_list.append([song, source, n/len(sources_list)])
	if n % len(sources_list):
		args_list[-1][2] += 1
	
	for args in args_list:
		pool(parse)(*args)
	
	gen = pool.iter()

	if returnGen:
		return gen
	return list(gen)

class BBQ(object): 

        def __init__(self,t=180,count=8):
		
                
            
            self.time = float(t)
            self.count = int(count)
            
            self.ThreadPool = ThreadPool(self.count)
        
        '''
    实际处理烧烤任务的函数
    '''
        def handle(self,task):
            
            time.sleep(self.time)#模拟烧烤时间
            try:
                task[0] = True
            except:
                pass
            return 
         
        '''
    添加一个烧烤任务
    task格式:[True/False],
    True代表处理完成
    False 代表等待处理
    '''
        def addTask(self,task):
        
            self.ThreadPool.addTask(self.handle,task)

def pickle_all_companies():
    tpool = ThreadPool(50)
    companies = Company.objects.all()
    for c in companies:
        tpool.add_task(pickle_company, c.symbol)
    tpool.wait_completion()

    return None

    def render_rap(self, msg_id, words):
        # Make the length of words fit the melody
        notes = sum(1 for pitch, beats in self._melody if pitch != REST)
        diff = notes - len(words)
        if diff < 0:
            words = words[:diff]
        else:
            words = words + ['la'] * diff

        delay = 0
        offsets = {}
        word_index = 0
        word_count = len(words)
        word_delays = []
        word_paths = []

        pool = ThreadPool(min(word_count, self._thread_pool))

        for pitch, beats in self._melody:
            duration = beats * self._spb

            if pitch != REST:
                word = words[word_index]
                word_delays.append(delay)
                word_path = '/tmp/%s-%s.wav' % (msg_id, word_index)
                word_paths.append(word_path)
                ssml = '<s><prosody pitch="%sHz" range="x-low">%s</prosody></s>' \
                    % (pitch, word)
                def task(word_id, ssml, word_path):
                    offsets[word_id] = self._swift.tts_file(ssml, word_path)
                pool.queue_task(task, (word_index, ssml, word_path))
                word_index += 1

            delay += duration

            if word_index == word_count:
                # Break here, rather than inside the if statement above, so that
                # that delay is updated and equals the duration of the rap.
                break

        pool.join_all()

        if not word_index:
            # Didn't render any words!
            return

        # Mix the rap and the backing track
        mix_path = '/tmp/%s-mix.wav' % msg_id
        sox_args = [self.sox_path, '-M'] + word_paths \
            + [self._backing_sample, mix_path, 'delay'] \
            + [str(delay + offsets[i]) for i, delay in enumerate(word_delays)] \
            + ['remix',
                ','.join(str(channel) for channel in range(1, word_count + 2)),
                'norm']
        print(' '.join(sox_args))
        subprocess.check_call(sox_args)

        return mix_path

 def startWork(self, work, argsList, resultCallback=None):
   try:
     requests = makeRequests(work, argsList, resultCallback, None)
     job = ThreadPool(self.threadNum)
     for req in requests:
       job.putRequest(req)
     job.wait()
   except:
     print sys.exc_info()

class ConcurrentTestPool(Singleton):
    @synchronized_self
    def init(self):
        self.pool = ThreadPool(multiprocessing.cpu_count())

    @synchronized_self
    def put(self, callable_, args=None, kwds=None):
        self.pool.putRequest(WorkRequest(callable_, args=args, kwds=kwds))

    def join(self):
        self.pool.wait()

def bfTest():
    pool = ThreadPool(100)
    for j in range(100):
        alltime = []
        for i in range(bingfa):
            work = WorkRequest(threads, args=(int(random.random() * portnum) % portnum,))
            pool.putRequest(work)
            sleep((1.0 / bingfa) * random.random())
            # threading.Thread(target=threads, args=(i % portnum,)).start()
        pool.wait()
        printdata(alltime)

    def prime_cache(self):
        """Ensures that the webpage cache is filled in the
        quickest time possible by making many requests in
        parallel"""

        print "Getting data for parts from suppliers' websites"
        pool = ThreadPool(NUM_THREADS)

        for srcode, pg in self.iteritems():
            print srcode
            pool.add_task(pg.get_price)

        pool.wait_completion()

def parse_soundcloud_api2(title):
	'''
	Function connects to soundcloud.com and returns the .mp3 links in it.
	
	API method 2: Parsing player's json data.
	'''
	links = search_soundcloud(title)
	
	pool = ThreadPool(max_threads=5, catch_returns=True, logger=log)
	for link in links:
		pool(get_soundcloud_dl_link)(link)
	
	return pool.iter()