Python Pool.map_async方法代码示例

# 需要导入模块: from multiprocessing.dummy import Pool [as 别名]
# 或者: from multiprocessing.dummy.Pool import map_async [as 别名]
 def Get_values(hosts,Redis_Key,keys,history,preset,operation):
     RC.delete(Redis_Key)
     hosts = [str(host[0]) for host in hosts]
     method = 'history.get'
     RC.set('%s_time' % Redis_Key, time.strftime('%Y-%m-%d %H:%M',time.localtime()))
     def Run(host):
         try:
             values = {}
             for key in keys:
                 value = zabbix_api.GET_value(host, keys[key], method, history)
                 if value != None:
                     if '.' in value:
                         value = float(value)
                     if operation == 'lt':
                         if int(value) < preset:
                             values[key] = value
                             RC.hset(Redis_Key, host, values)
                     if operation == 'gt':
                         if int(value) > preset:
                             values[key] = value
                             RC.hset(Redis_Key, host, values)
         except Exception as e:
             logging.error(e)
     pool = ThreadPool(4)
     pool.map_async(Run,hosts)
     pool.close()
     pool.join()

# 需要导入模块: from multiprocessing.dummy import Pool [as 别名]
# 或者: from multiprocessing.dummy.Pool import map_async [as 别名]
    def test_concurrent_writes(self):
        sender_pool = ThreadPool(100)
        receiver_pool = ThreadPool(100)

        message = can.Message(
            arbitration_id=0x123,
            extended_id=True,
            timestamp=121334.365,
            data=[254, 255, 1, 2]
        )
        workload = 1000 * [message]

        def sender(msg):
            self.bus1.send(msg)

        def receiver(_):
            result = self.bus2.recv(timeout=2.0)
            self.assertIsNotNone(result)
            self.assertEqual(result, message)

        sender_pool.map_async(sender, workload)
        receiver_pool.map_async(receiver, len(workload) * [None])

        sender_pool.close()
        sender_pool.join()
        receiver_pool.close()
        receiver_pool.join()

# 需要导入模块: from multiprocessing.dummy import Pool [as 别名]
# 或者: from multiprocessing.dummy.Pool import map_async [as 别名]
def update_advalue(update=False):
    ADValueTable.metadata.create_all(db)
    KDJValueTable.metadata.create_all(db)
    # idlist = sc.get_sza_index(index=0)
    # idlist.extend(sc.get_sha_index(index=0))
    idlist = sc.get_china_stock_list(index=0)
    # idlist = ['600777']
    # lock = threading.Lock()
    # for num in idlist:
    #     while threading.activeCount() >= 10:
    #         time.sleep(10)
    #     th = ADValueThread(lock, num, update)
    #     th.setDaemon(True)
    #     th.start()
    # for t in threading.enumerate():
    #     try:
    #         t.join()
    #     except Exception as ex:
    #         logger.debug('%s' % str(ex))
    #         continue
    lock = multiprocessing.Lock()
    tp = ThreadPool(10)
    # lock = multiprocessing.Manager().Lock()
    # tp = multiprocessing.Pool()
    tp.map_async(advalueprocess,
                 [(lock, num, update) for num in idlist]).get(36000)
    tp.close()
    tp.join()

# 需要导入模块: from multiprocessing.dummy import Pool [as 别名]
# 或者: from multiprocessing.dummy.Pool import map_async [as 别名]
 def thread_start(self, thread_run, args):
     if len(args) > 0:
         self.preAction(args)
         pool = Pool(self.cpu)
         pool.map_async(thread_run, args)
         pool.close()
         pool.join()

# 需要导入模块: from multiprocessing.dummy import Pool [as 别名]
# 或者: from multiprocessing.dummy.Pool import map_async [as 别名]
 def getvasprun(self, files):
     m = self.m
     maindir = pwd()
     if m.engine == "vasp":
         calculator = self.getVaspRun_vasp
     elif m.engine == "lammps":
         calculator = self.getVaspRun_lammps
     self.jm = jobManager()
     for file in files:
         print(file)
         dir = "dirs/dir_" + file
         mkdir(dir)
         mv(file, dir + '/POSCAR')
         cd(dir)
         calculator()
         cd(maindir)
     self.jm.run()
     if m.th:
         mkdir(m.pbsname)
         self.thcode(files, m.pbsname)
         cp("dirs", m.pbsname)
         passthru("tar zcf %s.tar.gz %s" % (m.pbsname, m.pbsname))
     print('start check')
     self.jm.check()
     if m.engine == "lammps1":
         from multiprocessing.dummy import Pool
         pool = Pool()
         pool.map_async(lammpsvasprun, files)
         pool.close()
         pool.join()

# 需要导入模块: from multiprocessing.dummy import Pool [as 别名]
# 或者: from multiprocessing.dummy.Pool import map_async [as 别名]
def test4():
    for n in range(100000):
        def test5(i):
            n += i
    tpool = TheaderPool(processes=1)
    tpool.map_async(test5,range(100000))
    tpool.close()
    tpool.join()

# 需要导入模块: from multiprocessing.dummy import Pool [as 别名]
# 或者: from multiprocessing.dummy.Pool import map_async [as 别名]
    def run_steps(self, steps=1):
        self.steps = steps
        
        pool = ThreadPool()
        pool.map_async(self._run, self.planners)

        pool.close()
        pool.join()

# 需要导入模块: from multiprocessing.dummy import Pool [as 别名]
# 或者: from multiprocessing.dummy.Pool import map_async [as 别名]
    def execute_moves(self, moves):
        def act(args):
            args[0](*args[1:])

        pool = ThreadPool(len(moves))
        pool.map_async(act, moves)

        pool.close()
        pool.join()

# 需要导入模块: from multiprocessing.dummy import Pool [as 别名]
# 或者: from multiprocessing.dummy.Pool import map_async [as 别名]
def get_taobao_seller(keywords):
    # 爬取指定数量的店铺信息
    def get_seller_from_num(nums):
        url = "https://shopsearch.taobao.com/search?data-key=s&data-value={0}&" \
              "ajax=true&_ksTS=1481770098290_1972&callback=jsonp602&app=shopsearch&q={1}&js=1&isb=0".format(
            nums, keywords)
        # url = "https://shopsearch.taobao.com/search?data-key=s&data-value={0}&ajax=true&callback=jsonp602&app=shopsearch&q={1}".format(nums,keywords)
        content = requests.get(url)
        wbdata = content.text[11:-2]
        data = json.loads(wbdata)
        shop_list = data['mods']['shoplist']['data']['shopItems']
        for s in shop_list:
            name = s['title']  # 店铺名
            nick = s['nick']  # 卖家昵称
            nid = s['nid']  # 店铺ID
            provcity = s['provcity']  # 店铺区域
            shopUrl = s['shopUrl']  # 店铺链接
            totalsold = s['totalsold']  # 店铺宝贝数量
            procnt = s['procnt']  # 店铺销量
            startFee = s['startFee']  # 未知
            mainAuction = s['mainAuction']  # 店铺关键词
            userRateUrl = s['userRateUrl']  # 用户评分链接
            dsr = json.loads(s['dsrInfo']['dsrStr'])
            goodratePercent = dsr['sgr']  # 店铺好评率
            srn = dsr['srn']  # 店铺等级
            category = dsr['ind']  # 店铺分类
            mas = dsr['mas']  # 描述相符
            sas = dsr['sas']  # 服务态度
            cas = dsr['cas']  # 物流速度
            data = {
                'name': name,
                'nick': nick,
                'nid': nid,
                'provcity': provcity,
                'shopUrl': shopUrl,
                'totalsold': totalsold,
                'procnt': procnt,
                'startFee': startFee,
                'goodratePercent': goodratePercent,
                # 'mainAuction':mainAuction,
                'userRateUrl': userRateUrl,
                'srn': srn,
                'category': category,
                'mas': mas,
                'sas': sas,
                'cas': cas
            }
            print(data)
            print("插入数据成功")

    pool = TheaderPool(processes=1)
    pool.map_async(get_seller_from_num, range(0, 10020, 20))
    pool.close()
    pool.join()

# 需要导入模块: from multiprocessing.dummy import Pool [as 别名]
# 或者: from multiprocessing.dummy.Pool import map_async [as 别名]
	def run(self):
		try:
			pool = ThreadPool(processes=self.thread)
			pool.map_async(self.start, self.ips).get(0xffff)
			pool.close()
			pool.join()
		except Exception as e:
			pass
		except KeyboardInterrupt:
			print u'\n{}[-] 用户终止扫描...{}'.format(R, W)
			sys.exit(1)
		finally:
			print '-'*55
			print u'{}[+] 扫描完成耗时 {} 秒.{}'.format(O, time.time()-self.time, W) 

# 需要导入模块: from multiprocessing.dummy import Pool [as 别名]
# 或者: from multiprocessing.dummy.Pool import map_async [as 别名]
    def main(self):
        self.s = self.login()
        self.keyword = input('请输入要搜的图的关键字：') #raw_input(u'请输入...')输入中文？
        print(self.keyword)
        rootPath = 'E:\ProgramCode\Python\pixivStar\\'+self.keyword
        if not os.path.exists(rootPath):
            os.mkdir(rootPath) #TODO 自定义路径
        #TODO arg参数，收藏数
        #get_search(keyword)
        searchUrl = self.index_url + '/search.php?s_mode=s_tag&word='+self.keyword+'&order=date_d&p='+str(1)
        respo = self.s.get(searchUrl, headers=self.header)
        content1 = respo.content.decode('utf-8')
        '''with open('contentExample.txt', 'w', encoding='utf-8') as f:
            print(content1,f)'''
        descriPat = re.compile('<meta name="description" content="(.*?)">', re.S)
        descri = re.findall(descriPat, content1)[0]
        print(descri)
        amountPat = re.compile('count-badge">(.*?)\u4ef6</span>', re.S)
        amount = int(re.findall(amountPat, content1)[0]) // 20
        print('页数：'+str(amount))
        '''self.thumbHeader = {'Host':'i2.pixiv.net', #host i1 ~ i4
            'If-Modified-Since':'Thu, 11 Aug 2016 01:15:38 GMT',
            'Referer':searchUrl,
            'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.106 Safari/537.36}'}

        signCheckPat = re.compile('signup-sina-button"(.*?)</button>')
        sign = re.findall(signCheckPat, content1)
        print(len(sign))'''
        '''signCheckPat2 = re.compile('bookmarks"(.*?)</li>')
        sign2 = re.findall(signCheckPat2, content1)
        print(len(sign2))'''
        #TODO 两种搜索模式
        #'s_mode=' 's_tag' #tag
        #'s_tc' #标题和介绍

        pool = ThreadPool(8)
        urls = []
        for page in range(1, amount):
            urls.append(self.index_url + '/search.php?word='+self.keyword+'&order=date_d&p='+str(page))
            #self.get_search(self.index_url + 'search.php?s_mode=s_tag&word='+self.keyword+'&order=date_d&p='+str(page))
        pool.map_async(self.get_search, urls)
        pool.close()
        pool.join()
        
        
        print('--mission complete--')
        #sys.stdout = io.TextIOWrapper(sys.stdout.buffer,encoding='utf-8')
        '''fileName = 'E:\ProgramCode\Python\searchPage.txt'

# 需要导入模块: from multiprocessing.dummy import Pool [as 别名]
# 或者: from multiprocessing.dummy.Pool import map_async [as 别名]
def multiRunuser():
    pool = ThreadPool(cpu_count() * 8)
    global ip_list
    global results
    results = pool.map_async(runuser, ip_list)
    pool.close()
    pool.join()

# 需要导入模块: from multiprocessing.dummy import Pool [as 别名]
# 或者: from multiprocessing.dummy.Pool import map_async [as 别名]
class BaseLogger:
    
    def __init__(self, outfile ='baselogger.log', sockets=[]):
        """
        A basic logging system.
        :param outfile: The file the logging output will be directed to.
        :param sockets: A list of external sockets who wish to be notified
        
        """
        self.outfile = 'baselogger.out'
        self.fh = open(outfile, 'a')
        self.sockets = sockets
        self.level_text = ['INFO', 'WARNING', 'ERROR']
        self.pool = Pool(2 if self.sockets else 1)
    
            
    def Log(self, message, level=0):
        """
        Fires off a job to write log information to file and the listening sockets.
        """
        timestamp = strftime("%Y-%m-%d %H:%M:%S")
        format_message = "# {0} [{1}]\t: {2}".format(timestamp, self.level_text[level], message)
        if level == 1:
            color = Color.YELLOW
        elif level == 2:
            color = Color.RED
        else:
            color = Color.GREEN
            
            
        sys.stdout.write(color + format_message + Color.END + "\n")
        self.pool.map_async(_log_fs, [(self.fh, format_message)] )
        self.pool.map_async(_log_socket, [(s, format_message) for s in self.sockets])
        
      
    def destroy(self):
       """
       Clean up resources.  Closes all open files and sockets.
       """
       self.Log("Destroying logging system.", level=1)
       self.pool.close()
       # Continue until all operations have finished
       self.fh.close()
       for s in self.sockets:
          s.close()
       self.pool.close()

# 需要导入模块: from multiprocessing.dummy import Pool [as 别名]
# 或者: from multiprocessing.dummy.Pool import map_async [as 别名]
def main():
    #TODO clean Logs
    parser = argparse.ArgumentParser(prog='scan.py', description='Pentesting Scan')
    parser.add_argument('-t', dest='TARGET', required=True, help='192.168.1.0/24 or 192.168.1.0-128') #required
    parser.add_argument('-d', dest='DIR', default='results', help='specifiy output-directory')#optional
    args = parser.parse_args()
    
    global logdir
    logdir = args.DIR
    
    checkEnv(args.DIR)
    global targets
    targets = netscan(args.TARGET)
    global db
    db=dict()
    
 
    portscanPool = Pool(4)
    portscanResults = portscanPool.map_async(portscan, targets)
    portscanPool.close()
    portscanPool.join()
    
    ipServiceList= []
    
    
    for ip in db.keys():
        if len(db[ip][1]) == 0 and len(db[ip][0]) == 0:
            print "[INFO ] no open ports in %s, delete key and remove dir" % ip
            del db[ip]
            removeEmptyEnv(ip)
        else:
            for s in range(len(db[ip][1])):
                ipServiceList.append([ip, s])
                
    
    sericePool = Pool(4)
    serviceResults = sericePool.map_async(checkService, ipServiceList)
    sericePool.close()
    sericePool.join()
    
    print "[ END ] "+"="*35+" [ END ]"

# 需要导入模块: from multiprocessing.dummy import Pool [as 别名]
# 或者: from multiprocessing.dummy.Pool import map_async [as 别名]
class IoTaskThreadPool(object):
    '''
       thread pool for io operations
    '''
    def __init__(self, poolsize):
        self.ioPool = ThreadPool(poolsize)

    def execTasks(self, ioFunc, ioParams):
        if not ioParams or len(ioParams) == 0:
            return []
        return self.ioPool.map(ioFunc, ioParams)

    def execTasksAsync(self, ioFunc, ioParams):
        if not ioParams or len(ioParams) == 0:
            return []
        self.ioPool.map_async(ioFunc, ioParams)

    def close(self):
        self.ioPool.close()

    def join(self):
        self.ioPool.join()