Python dummy.Pool方法代码示例

# 需要导入模块: from multiprocessing import dummy [as 别名]
# 或者: from multiprocessing.dummy import Pool [as 别名]
def run(self):
        # http://www.kuaidaili.com/proxylist/1/
        pool = Pool()
        tt = pool.map(self.my_run, [page for page in xrange(1, 11)])
        t_result = []
        for x in tt:
            t_result += x

        # 填充结果集
        result = []
        info = dict()
        info['url'] = self.url
        info['type'] = self.type
        info['tag'] = self.tag
        result.append(info)
        result.append(t_result)
        self.result_queue.put(result) 

# 需要导入模块: from multiprocessing import dummy [as 别名]
# 或者: from multiprocessing.dummy import Pool [as 别名]
def __init__(self, feats_path, target, n_frames_per_video, batch_size, n_feat_maps, feat_map_side_dim, n_threads=10, annotation_dict=None):
        random.seed(101)
        np.random.seed(101)

        self.__feats_pathes = feats_path
        self.__n_frames_per_video = n_frames_per_video
        self.__n_feat_maps = n_feat_maps
        self.__feat_map_side_dim = feat_map_side_dim
        self.__annotation_dict = annotation_dict

        self.__batch_size = batch_size
        self.__y = target

        self.__is_busy = False
        self.__batch_features = None
        self.__batch_y = None
        self.__n_threads_in_pool = n_threads
        self.__pool = Pool(self.__n_threads_in_pool) 

# 需要导入模块: from multiprocessing import dummy [as 别名]
# 或者: from multiprocessing.dummy import Pool [as 别名]
def _start(self):
		try:
			print '-'*60
			print u'{}[-] 正在扫描地址: {}{} '.format(self.O, 
				socket.gethostbyname(self.target), self.W)
			print '-'*60
			#线程数
			pool = ThreadPool(processes=100)
			#get传递超时时间，用于捕捉ctrl+c
			pool.map_async(self.run, self.ports).get(0xffff)
			pool.close()
			pool.join()
			print '-'*60
			print u'{}[-] 扫描完成耗时: {} 秒.{}'.format(self.O, 
				time()-self.time, self.W)
		except Exception as e:
			print e
		except KeyboardInterrupt:
			print self.R + u'\n[-] 用户终止扫描...'
			sys.exit(1) 

# 需要导入模块: from multiprocessing import dummy [as 别名]
# 或者: from multiprocessing.dummy import Pool [as 别名]
def get_node_id_feature_sparse(self,X):


        pool = ThreadPool(40)
        #results = map(self.get_feaure, np.array(X.values))
        results = pool.map(self.get_feaure, np.array(X.values))

        results = list(results)
        #print(results)
        #results = np.array(results)
        #print(results)
        results = pd.DataFrame(results)

        print(results.columns)
        print("-------------")
        results = pd.SparseDataFrame(pd.get_dummies(results)).astype("float")



        print(results)

        # columns = results.columns
        # results = scipy.sparse.csr_matrix(results)
        print(results.columns)
        return results 

# 需要导入模块: from multiprocessing import dummy [as 别名]
# 或者: from multiprocessing.dummy import Pool [as 别名]
def reprojectToThisThreaded(self, sourceProjection, numThreads):
    uvList = []
    fx = float(self.imsize[0])
    fy = float(self.imsize[1])

    angleList = [self.angular_position((float(i)/fx,float(j)/fy)) for i in range(self.imsize[0]) for j in range(self.imsize[1])]

    poolAngles = ThreadPool(numThreads)
    image = poolAngles.map(sourceProjection.pixel_value, angleList)
    poolAngles.close()
    poolAngles.join()

    idx = 0
    for x in range(self.imsize[0]):
      for y in range(self.imsize[1]):
        pixel = image[idx]
        if pixel is None:
          print(x,y)
        else:
          self.image[y,x] = pixel
        idx = idx + 1 

# 需要导入模块: from multiprocessing import dummy [as 别名]
# 或者: from multiprocessing.dummy import Pool [as 别名]
def __init__(self, args):
        super(Slave, self).__init__()
        self._pool=Pool(args.thread_num)
        self._timeout=args.int_timeout
        self._call_method=getattr(requests,args.int_method)
        self._flags=args.int_flags.split(',,')
        if args.int_headers!="":
            self._headers=json.loads(input2json(args.int_headers))
        else:
            self._headers={}

        if args.int_cookies!='':
            cookiesl=args.int_cookies.split(',')
            self._cookies={x.split(':')[0]:x.split(':')[1] for x in cookiesl}
        else:
            self._cookies={} 

# 需要导入模块: from multiprocessing import dummy [as 别名]
# 或者: from multiprocessing.dummy import Pool [as 别名]
def ancestral_reconstruction(outgroup_genes):
    """ Use Lazarus wrapper to run paml ancestral reconstruction """
    print "Ancestral Reconstruction"
    def run_lazarus(outgroup_gene):
        coreGene, ingroup, outgroup = outgroup_gene
        align = "%s/alignment/%s.nt_ali.fasta" % (coreGene, coreGene)
        tree = "%s/tree/RAxML_bestTree.ml_%s.nt_ali" % (coreGene, coreGene)
        model = "/opt/PepPrograms/paml4.8/dat/wag.dat"
        outdir = os.path.abspath("%s/ancestral/" % coreGene)
        call_with_log(LAZARUS_PATH + " --codeml --outputdir %s --verbose 9 \
--alignment %s --tree %s --model %s --asrv 4 --gapcorrect --getanc --ingroup %s\
 --outgroup %s" % (outdir, align, tree, model, "[%s]" % (",".join(ingroup)), 
"[%s]" % (outgroup)))
    pool = ThreadPool(args.threads)
    pool.map(run_lazarus, outgroup_genes) 
    pool.close()
    pool.join() 

# 需要导入模块: from multiprocessing import dummy [as 别名]
# 或者: from multiprocessing.dummy import Pool [as 别名]
def test_multiprocessing(self):
        from multiprocessing.dummy import Pool
        import random
        import time

        def square_slowly(x):
            time.sleep(random.uniform(0.1, 0.2))
            return x ** 2

        pool = Pool(8)
        start = time.time()
        self.assertEqual([0, 1, 4, 9, 16, 25, 36, 49],
                         pool.map(square_slowly, range(8), chunksize=1))
        duration = time.time() - start
        self.assertGreater(duration, 0.1)
        self.assertLess(duration, 0.25) 

# 需要导入模块: from multiprocessing import dummy [as 别名]
# 或者: from multiprocessing.dummy import Pool [as 别名]
def recursive(self, my_dir):
        self.print_info("Recursive mode")
        files_suid = []
        files_sgid = []
        files_list = []
        for cpwd, dirs, files in walk(my_dir):
            if cpwd.endswith("/"):
                cwd = cpwd
            else:
                cwd = cpwd + "/"
            for f in files:
                files_list.append(cwd + f)
        pool = Pool(8)
        results = pool.map(self.is_suid_sgid, files_list)
        pool.close()
        pool.join()
        for result in results:
            if result[0]:
                files_suid.append(result[0])
            if result[1]:
                files_sgid.append(result[1])

        return [files_suid, files_sgid] 

# 需要导入模块: from multiprocessing import dummy [as 别名]
# 或者: from multiprocessing.dummy import Pool [as 别名]
def deploy_marvin(self):
        if not self.get_marvin_json():
            return False
        print("Note: Found hypervisor type '" + self.get_hypervisor_type() + "'")
        vms = []
        for zone in self.get_zones():
            for pod in self.get_pods(zone=zone):
                for cluster in self.get_clusters(pod=pod):
                    vms += self.get_hosts(cluster=cluster)
                vms += self.get_management_hosts(zone=zone)
        vms += self.get_nsx_nodes()
        thread_number = 10
        if self.running_on_vm:
            thread_number = 4
        pool = ThreadPool(thread_number)
        results = pool.map(self.deploy_host, vms)
        print("Note: Deployment results: " + str(results))
        pool.close()
        pool.join()
        return True

    # Delete Marvin infra 

# 需要导入模块: from multiprocessing import dummy [as 别名]
# 或者: from multiprocessing.dummy import Pool [as 别名]
def delete_marvin(self):
        if not self.get_marvin_json():
            return False
        print("Note: Found hypervisor type '" + self.get_hypervisor_type() + "'")
        hosts = self.get_hosts()
        thread_number = 10
        if self.running_on_vm:
            thread_number = 4
        pool = ThreadPool(thread_number)
        results = pool.map(self.delete_host, hosts)
        print("Note: Deployment results: " + str(results))
        pool.close()
        pool.join()
        return True

    # VM or not? 

# 需要导入模块: from multiprocessing import dummy [as 别名]
# 或者: from multiprocessing.dummy import Pool [as 别名]
def download_dataset(all_tasks, num_workers=4):
    def urlretrieve_star(args):
        return urlretrieve(*args)

    pool = Pool(num_workers)
    pool.map(urlretrieve_star, all_tasks)
    pool.close()
    pool.join() 

# 需要导入模块: from multiprocessing import dummy [as 别名]
# 或者: from multiprocessing.dummy import Pool [as 别名]
def do_upserts(self, account_list, concurrency, upsert_func):
        """Runs the upsert command for the accounts in `account_list`. Execution
        happens in concurrent processes."""

        success_ctr = multiprocessing.Value('i', 0, lock=True)
        retry_ctr = multiprocessing.Value('i', 0, lock=True)

        def _updater(acct):
            upsert_func(addr=self.TEST_SERVER_ADDR, account=acct,
                        success_ctr=success_ctr, retry_ctr=retry_ctr)

        pool = mpd.Pool(concurrency)
        results = [
            pool.apply_async(_updater, (acct,))
            for acct in account_list for _ in range(concurrency)
        ]

        _ = [res.get() for res in results]
        pool.close() 

# 需要导入模块: from multiprocessing import dummy [as 别名]
# 或者: from multiprocessing.dummy import Pool [as 别名]
def generate_k_clusters(self,folder):
    pool = ThreadPool(cpu_count())
    result = pool.map(self.read_image, folder)
    pool.close()
    pool.join()
    self.cluster = [r[0] for r in result if r[0] != None]
    self.data = [r[1] for r in result if r[1] != None]
    self.end = [r[2] for r in result if r[2] != None] 

# 需要导入模块: from multiprocessing import dummy [as 别名]
# 或者: from multiprocessing.dummy import Pool [as 别名]
def __init__(self, dataset, feedin_shape, collate_fn=default_collate, threads=1, shuffle=False):
        super(DataLoader, self).__init__()

        self.dataset = dataset
        self.threads = threads
        self.collate_fn = collate_fn(feedin_shape)
        # self.collate_fn = self.default_collate_fn

        # shape related variables

        self.data_shapes = feedin_shape['data']
        self.label_shapes = feedin_shape['label']
        self.batch_size = feedin_shape['batch_size']

        # loader related variables
        self.current = 0
        self.total = len(self.dataset)
        self.shuflle = shuffle
        self.map_index = list(range(self.total))

        # prepare for loading
        self.get_batch = self.get_batch_single_thread
        if self.threads > 1:  # multi process read
            from multiprocessing.dummy import Pool as ThreadPool
            # self.pool = multiprocessing.Pool(self.threads)
            self.pool = ThreadPool(self.threads)
            self.get_batch = self.get_batch_multi_thread

        self.reset()