Python ThreadPool.iter方法代码示例

# 需要导入模块: from threadpool import ThreadPool [as 别名]
# 或者: from threadpool.ThreadPool import iter [as 别名]
def search(song, n, processes=config.search_processes, returnGen=False):
	'''
	Function searches song and returns n valid .mp3 links.
	@param song: Search string.
	@param n: Number of songs.
	@param processes: Number of processes to launch in the subprocessing pool.
	@param returnGen: If true, a generator of the links will be returned,
						and not the calculated list itself.
	'''
	sources_list = [x for x in config.search_sources_const if config.search_sources[x]]
	log.debug("Using sources: %s" % sources_list)
	
	# IMPROVE: better handeling of slicing.
	pool = ThreadPool(max_threads=min(processes, len(sources_list)), catch_returns=True, logger=log)
	args_list = []
	for source in sources_list:
		args_list.append([song, source, n/len(sources_list)])
	if n % len(sources_list):
		args_list[-1][2] += 1
	
	for args in args_list:
		pool(parse)(*args)
	
	gen = pool.iter()

	if returnGen:
		return gen
	return list(gen)

# 需要导入模块: from threadpool import ThreadPool [as 别名]
# 或者: from threadpool.ThreadPool import iter [as 别名]
def parse_soundcloud_api2(title):
	'''
	Function connects to soundcloud.com and returns the .mp3 links in it.
	
	API method 2: Parsing player's json data.
	'''
	links = search_soundcloud(title)
	
	pool = ThreadPool(max_threads=5, catch_returns=True, logger=log)
	for link in links:
		pool(get_soundcloud_dl_link)(link)
	
	return pool.iter()

# 需要导入模块: from threadpool import ThreadPool [as 别名]
# 或者: from threadpool.ThreadPool import iter [as 别名]
def parse_bandcamp(title):
	links = search_bandcamp(title)
	max_result_parsing = 3
	i = 0
	
	pool = ThreadPool(max_threads=5, catch_returns=True, logger=log)
	for link in links:
		if '/album/' in link:
			pool(get_bandcamp_album_dl_links)(link)
			i += 1
			
		elif '/track/' in link:
			pool(get_bandcamp_dl_link)(link)
			i += 1
		
		if i >= max_result_parsing:
			break
			
	return pool.iter()

# 需要导入模块: from threadpool import ThreadPool [as 别名]
# 或者: from threadpool.ThreadPool import iter [as 别名]
	def run(self): # Called by Qt once the thread environment has been set up.
		google_ans = Main.WebParser.WebServices.googleImageSearch(self.song)[:]
		pool = ThreadPool(max_threads=config.GoogleImagesGrabber_processes, catch_returns=True, logger=log)
		
		fn_list = []
		while len(fn_list) < self.numOfPhotos and google_ans:
			urls = []
			for i in range(self.numOfPhotos-len(fn_list)):
				if google_ans:
					urls.append(google_ans.pop(0))

			for url in urls:
				pool(self.fetchPhoto)(url)
				
			for photo in pool.iter():
				try:
					if photo:
						fn_list.append(photo)
				except Exception, e:
					log.warning("Exception %s ignored in GoogleImagesGrabberThread." % str(e))

# 需要导入模块: from threadpool import ThreadPool [as 别名]
# 或者: from threadpool.ThreadPool import iter [as 别名]
	def run(self): # Called by Qt once thethread environment has been set up.
		if urlparse(self.song).scheme in config.allowd_web_protocols:
			# if a url and not a search string
			self.isDirectLink = True
			self.url = self.song
			self.song = ""
			domainName = urlparse(self.url).netloc.lower()
			
			if domainName.endswith('youtube.com') or domainName.endswith('youtu.be'):
				queries = parse_qs(urlparse(self.url).query)
				if 'p' in queries or 'list' in queries:
					log.debug("Url is a direct url (Youtube playlist)")
					if 'p' in queries:
						playlist_id = queries['p'][0]
					elif 'list' in queries:
						playlist_id = queries['list'][0]
					videos_ids = Main.WebParser.LinksGrabber.parse_Youtube_playlist(playlist_id)	
					
					t_pool = ThreadPool(max_threads=config.buildSongObjs_processes, catch_returns=True, logger=log)
					for id in videos_ids:
						t_pool(Main.WebParser.LinksGrabber.get_youtube_dl_link)(id)
					links_gen = t_pool.iter()
					
				else:
					log.debug("Url is a direct url (Youtube)")
					if domainName.endswith('youtube.com'):
						video_id = queries['v'][0]
					else:
						video_id = urlparse(self.url).path.strip('/')
						
					try:
						metaUrlObj = Main.WebParser.LinksGrabber.get_youtube_dl_link(video_id)
					except YoutubeException, e:
						self.error.emit(e)
						
					links_gen = (x for x in [metaUrlObj])
			elif domainName.endswith('bandcamp.com'):
				if '/album/' in self.url:
					log.debug("Url is a direct url (bandcamp album)")
					metaUrlObjs = Main.WebParser.LinksGrabber.get_bandcamp_album_dl_links(self.url)
					links_gen = (x for x in metaUrlObjs)
				elif '/track/' in self.url:
					log.debug("Url is a direct url (bandcamp)")
					metaUrlObj = Main.WebParser.LinksGrabber.get_bandcamp_dl_link(self.url)
					links_gen = (x for x in [metaUrlObj])
				else:
					links_gen = (x for x in [])
			elif domainName.endswith('soundcloud.com'):
				log.debug("Url is a direct url (Soundcloud)")
				if self.url.startswith('https://'):
					self.url = self.url.replace('https://', 'http://')
				metaUrlObj = Main.WebParser.LinksGrabber.get_soundcloud_dl_link(self.url)	
				links_gen = (x for x in [metaUrlObj])
			else:
				ext = self.url.split('/')[-1].split('.')[-1]
				if re.match(r"^http://.*soundcloud\.com/.+/.+/download$", self.url):
					log.debug("Url is a direct url (Soundcloud).")
					metaUrlObj = utils.classes.MetaUrl(self.url, "Direct Link")
				elif ext in ['mp3', 'mp4', 'flv', 'webm']:
					log.debug("Url is a direct url (%s file)." % ext)
					metaUrlObj = utils.classes.MetaUrl(self.url, "Direct Link")
				elif ext:
					log.debug("Url is a direct url (%s - Non-multimedia file)." % ext)
					metaUrlObj = utils.classes.MetaUrl(self.url, "Direct Non-Multimedia Link")
				else:
					log.debug("Url is a direct url, no extention provided.")
					log.error("got NotSupportedFiletypeException() for the \"%s\" extention." % ext)
					self.error.emit(NotSupportedFiletypeException(ext))
					return
					
				links_gen = (x for x in [metaUrlObj])