Python Manager.list方法代码示例

# 需要导入模块: from multiprocessing import Manager [as 别名]
# 或者: from multiprocessing.Manager import list [as 别名]
def sync():
    from multiprocessing import Manager
    from common import bounty, settings, peers
    from common.safeprint import safeprint
    man = Manager()
    items = {'config':man.dict(),
             'peerList':man.list(),
             'bountyList':man.list(),
             'bountyLock':bounty.bountyLock,
             'keyList':man.list()}
    items['config'].update(settings.config)
    items['peerList'].extend(peers.peerlist)
    items['bountyList'].extend(bounty.bountyList)
    safeprint(items)
    safeprint(items.get('bountyList'))
    safeprint(items.get('keyList'))
    if items.get('config') is not None:
        from common import settings
        settings.config = items.get('config')
    if items.get('peerList') is not None:
        global peerList
        peers.peerlist = items.get('peerList')
    if items.get('bountyList') is not None:
        from common import bounty
        bounty.bountyList = items.get('bountyList')
    if items.get('bountyLock') is not None:
        from common import bounty
        bounty.bountyLock = items.get('bountyLock')
    return items

# 需要导入模块: from multiprocessing import Manager [as 别名]
# 或者: from multiprocessing.Manager import list [as 别名]
def run():
    # build the mdp
    start = time.time()
    room_size = 3
    num_rooms = 5
    mdp = maze_mdp.MazeMDP(room_size=room_size, num_rooms=num_rooms)

    # build the agent
    m = Manager()
    init_dict = {(s, a): 0 for s in mdp.states for a in mdp.ACTIONS + [None]}
    shared_weights = m.dict(init_dict)
    shared_value_weights = m.dict(init_dict)
    agent = async_actor_critic.AsyncActorCritic(actions=mdp.ACTIONS, discount=mdp.DISCOUNT, 
        weights=shared_weights, value_weights=shared_value_weights, tau=.3, learning_rate=.5)

    # build a single experiment
    rewards = m.list()
    start_state_values = m.list()
    max_steps = (2 * room_size * num_rooms) ** 2
    exp = experiment.Experiment(mdp=mdp, agent=agent, num_episodes=800, max_steps=max_steps,
        rewards=rewards, start_state_values=start_state_values)

    # run the experiment
    multiexperiment = experiment.MultiProcessExperiment(experiment=exp, num_agents=NUM_PROCESSES)
    multiexperiment.run()

    # report results
    end = time.time()
    print 'took {} seconds to converge'.format(end - start)
    mdp.print_state_values(shared_value_weights)
    optimal = mdp.EXIT_REWARD + (2 * room_size * num_rooms * mdp.MOVE_REWARD)
    utils.plot_values(rewards, optimal, 'rewards')
    utils.plot_values(start_state_values, optimal, 'start state value')

# 需要导入模块: from multiprocessing import Manager [as 别名]
# 或者: from multiprocessing.Manager import list [as 别名]
    def test_tcpc_tcps(self):

        conn = {'ip': '127.0.0.1', 'port': self.port1}
        N = 10
        manager = Manager()
        ls = manager.list(self.test_msg)
        ls_out = manager.list()
        dc = manager.dict(conn)
        os.system('fuser -k '+conn['port']+'/tcp')
        processes = [Process(target=tcpserv.tcp_serv, args=(dc,)),]
        for i in range(N):
            processes.append(Process(target=tcpcl.tcp_cl, args=(dc, ls, ls_out,)))

        processes[0].start()
        time.sleep(0.1)
        for p in processes[1:]:
            p.start()
            time.sleep(0.1)

        now = time.time()
        while N*len(self.test_msg) != len(ls_out) and \
          (now+30 > time.time()):
            time.sleep(1)
        print("tcp = %.3f" %(time.time() - now))

        processes.reverse()
        for p in processes:
            p.terminate()
            p.join()

        os.system('fuser -k '+conn['port']+'/tcp')
        #print(ls_out)
        print('tcp msg count %i' %len(ls_out))
        self.assertTrue(N*len(self.test_msg) == len(ls_out))

# 需要导入模块: from multiprocessing import Manager [as 别名]
# 或者: from multiprocessing.Manager import list [as 别名]
    def searchDic(self, items):
        # print('step0')
        managerlist = Manager()
        # print('step1')
        list_existed = managerlist.list()
        list_notExisted = managerlist.list()
        # print('step2')
        procs = []

        for i in range(len(items)):
            p = multiprocessing.Process(target=self.searchDicInner, args=(items[i], list_existed, list_notExisted))
            procs.append(p)
            p.start()

        # wait for all work process to finish
        for p in procs:
            p.join()

        self.__quickSort(list_existed)

        # print(list_existed)
        # print(list_notExisted)
        rtn = {}
        rtn["existed"] = list_existed[0:]
        rtn["notExisted"] = list_notExisted[0:]
        return rtn

# 需要导入模块: from multiprocessing import Manager [as 别名]
# 或者: from multiprocessing.Manager import list [as 别名]
def parallel_file_list(fet_dir,num_tasks):
    file_list = []
    for root,dirs,files in os.walk(fet_dir):
        for name in files:
            fet_file = os.path.join(root,name)
            file_list.append(fet_file)
    #process_file_list(file_list,1)
    num_files_per_task = int(len(file_list)/num_tasks)
    worker_list = []
    object_manager = Manager(); 
    for task in range(0,num_tasks):
        current_file_list = []
        if task < num_tasks - 1:
            current_file_list = object_manager.list(file_list[task*num_files_per_task:(task+1)*num_files_per_task])
        else:
            current_file_list = object_manager.list(file_list[task*num_files_per_task:])
        
        worker = Process(target=process_file_list,args=(current_file_list,task))
        worker.start()
        worker_list.append(worker)
        print 'Task %d invoked.'%task    
    
    for worker in worker_list:
        worker.join()
        print 'one worker exit.'

# 需要导入模块: from multiprocessing import Manager [as 别名]
# 或者: from multiprocessing.Manager import list [as 别名]
def controller_failure_unit_test():
    s = ["1001"]
    s1 = ["1002"]
    clear_config(s)
    clear_config(s1)
    manager1 = Manager()
    manager2 = Manager()
    failure1 = manager1.Value('i', 0)
    failed_list1 = manager1.list([])

    failure2 = manager2.Value('i', 0)
    failed_list2 = manager2.list([])
    processes = []
    process2 = mp.Process(target=controller_failure_detection, args=(s, '1', failure1, failed_list1,))
    processes.append(process2)
    process4 = mp.Process(target=controller_failure_detection, args=(s, '2', failure2, failed_list2,))
    processes.append(process4)
    for p in processes:
        p.start()
        print 'STARTING:', p, p.is_alive()
    r = random.randint(1, 10)
    time.sleep(r)
    print 'terminated'
    t1 = time.time()
    logging.debug(str( ["controller failed at:"] + [t1]))
    processes[0].terminate()
# Exit the completed processes
    for p in processes:
        p.join()
        print 'JOINED:', p, p.is_alive()

# 需要导入模块: from multiprocessing import Manager [as 别名]
# 或者: from multiprocessing.Manager import list [as 别名]
def start(config: Config):
    set_session_config(per_process_gpu_memory_fraction=1, allow_growth=True, device_list=config.opts.device_list)
    base_model = {'digest': 'd6fce85e040a63966fa7651d4a08a7cdba2ef0e5975fc16a6d178c96345547b3', 'elo': 0}
    m = Manager()
    base_weight_path = os.path.join(config.resource.next_generation_model_dir, base_model['digest'] + '.h5')
    model_base = load_model(config, config.resource.model_best_config_path, base_weight_path)
    modelbt_pipes = m.list([model_base.get_pipes(need_reload=False) for _ in range(config.play.max_processes)])
    
    while True:
        while not check_ng_model(config, exculds=[base_model['digest'] + '.h5']):
            logger.info(f"Next generation model is None, wait for 300s")
            sleep(300)

        logger.info(f"Loading next generation model!")
        digest = check_ng_model(config, exculds=[base_model['digest'] + '.h5'])
        ng_weight_path = os.path.join(config.resource.next_generation_model_dir, digest + '.h5')
        model_ng = load_model(config, config.resource.next_generation_config_path, ng_weight_path)
        modelng_pipes = m.list([model_ng.get_pipes(need_reload=False) for _ in range(config.play.max_processes)])

        # play_worker = EvaluateWorker(config, model1_pipes, model2_pipes)
        # play_worker.start()
        with ProcessPoolExecutor(max_workers=config.play.max_processes) as executor:
            futures = []
            for i in range(config.play.max_processes):
                eval_worker = EvaluateWorker(config, modelbt_pipes, modelng_pipes, pid=i)
                futures.append(executor.submit(eval_worker.start))
        
        wait(futures)
        model_base.close_pipes()
        model_ng.close_pipes()

        results = []
        for future in futures:
            results += future.result()
        base_elo = base_model['elo']
        ng_elo = base_elo
        for res in results:
            if res[1] == -1: # loss
                res[1] = 0
            elif res[1] != 1: # draw
                res[1] = 0.5
            if res[0] % 2 == 0:
                # red = base
                _, ng_elo = compute_elo(base_elo, ng_elo, res[1])
            else:
                # black = base
                ng_elo, _ = compute_elo(ng_elo, base_elo, 1 - res[1])
        logger.info(f"Evaluation finished, Next Generation's elo = {ng_elo}, base = {base_elo}")
        # send ng model to server
        logger.debug(f"Sending model to server")
        send_model(ng_weight_path)
        data = {'digest': digest, 'elo': ng_elo}
        http_request(config.internet.add_model_url, post=True, data=data)
        os.remove(base_weight_path)
        base_weight_path = ng_weight_path
        base_model['disgest'] = digest
        base_model['elo'] = ng_elo
        model_base = model_ng
        modelbt_pipes = m.list([model_base.get_pipes(need_reload=False) for _ in range(config.play.max_processes)])

# 需要导入模块: from multiprocessing import Manager [as 别名]
# 或者: from multiprocessing.Manager import list [as 别名]
 def __init__(self):
     manager=Manager()
     self.package=manager.dict()
     self.changeData=manager.list()
     self.changeDataEnco=manager.list()
     self.initPackage()
     self.leftEnco=0
     self.rightEnco=0

# 需要导入模块: from multiprocessing import Manager [as 别名]
# 或者: from multiprocessing.Manager import list [as 别名]
def upload_directory(app):
    """ Uploads an entire local directory. """
    manager = Manager()
    directories = manager.list()
    files = manager.list()
    remote_objects = manager.dict()
    exit_code = 0

    app.authenticate()

    logging.debug("%s %s", app.token, app.url)

    logging.info("Starting harvesters")
    local = Process(target=catalog_directory,
            args=(app, files, directories,))
    remote = Process(target=catalog_remote,
        args=(app, remote_objects,))

    remote.start()
    local.start()

    logging.info("Waiting for harvest")
    local.join()
    remote.join()

    backlog = delta_force_one(files, directories, remote_objects,
            prefix=app.prefix)

    logging.debug("Backlog: %s", backlog)
    if app.threads:
        p = Pool(processes=app.threads)
        # remove client property as it can't be pickled
        app.client = None
        try:
            rs = p.map_async(app, backlog, 1)
            p.close()
            rs.wait()
            if not rs.successful():
                raise rs.get()

            p.join()
            codes = rs.get()
        except KeyboardInterrupt:
            logging.info("Trying to stop...")
            p.terminate()
            return 130
    else:
        codes = map(app, backlog)

    logging.info("Done backing up %s to %s", app.source, app.container)

    if any(codes):
        logging.warn("Backup completed, but with errors. Check the log")
        exit_code = 1

    return exit_code

# 需要导入模块: from multiprocessing import Manager [as 别名]
# 或者: from multiprocessing.Manager import list [as 别名]
    def ComputePolicyParallel(self):

        print "Computing full policy"

        core_worker = cpu_count()
        print "Creating %d workers" % (core_worker)

        # TODO find a better way!
        # data=copy.deepcopy(self.Transition.probs.probs)
        # self.datacopy=data
        self.datacopy = copy.deepcopy(self.Transition.probs.probs)

        in_queue_list = []
        # res_queue_list=[] #a list of output queue

        manager = Manager()
        max_policy = manager.list(np.zeros(self.G.num_vertices()))
        full_policy = manager.list([[] for i in range(self.G.num_vertices())])

        for i in range(core_worker):

            in_queue_list.append([])

        # cut the total range of element into subranges for each worker
        split_range = np.array_split(
            np.array(range(self.G.num_vertices())), core_worker)

        # we put in each input queue the list of vertices to be computed
        for i in range(core_worker):
            r = split_range[i]
            for v in r:
                in_queue_list[i].append((self.Vertices_pos[v]))

        workers = [Process(target=self.PolicyWorker, args=(i, in_queue_list[
                           i], self.datacopy, max_policy, full_policy)) for i in range(core_worker)]

        for each in workers:
            # print "Starting Worker"
            each.start()

        print "\tJoin() ?"
        for each in workers:
            each.join()
            print "\tJoin() done"

        print "Policy done"

        self.max_policy = np.array(max_policy)
        self.p_u = np.array(full_policy)

        del max_policy
        del full_policy
        return self.max_policy

# 需要导入模块: from multiprocessing import Manager [as 别名]
# 或者: from multiprocessing.Manager import list [as 别名]
def sync():
    from multiprocessing import Manager
    man = Manager()
    items = {'config': man.dict(),
             'peerList': man.list(),
             'bountyList': man.list(),
             'bountyLock': bounty.bountyLock,
             'keyList': man.list()}
    items['config'].update(settings.config)
    items['peerList'].extend(peers.peerlist)
    items['bountyList'].extend(bounty.bountyList)
    safeprint(items)
    peers.sync(items)
    return items

# 需要导入模块: from multiprocessing import Manager [as 别名]
# 或者: from multiprocessing.Manager import list [as 别名]
    def test_tcpc_twproxy_tcps(self):

        conn = {
            'client': {'ip': '127.0.0.1', 'port':  self.port1}, # client -> tw_proxy
            'server': {'ip': '127.0.0.1', 'port':  self.port2},  # proxy -> tw_server
            } 
        N = 10
        manager = Manager()
        ls = manager.list(self.test_msg)
        ls_out = manager.list()

        dc = {}

        for d, k in zip([c for c in conn], [conn[c] for c in conn]):
            dc[d]  = manager.dict(k)
            os.system('fuser -k '+conn[d]['port']+'/tcp')

        processes = []
        processes.append(Process(target=tcpserv.tcp_serv, args=(dc['server'],))) # server
        proxy_args = {'port_in': self.port1, 'ip_out': '127.0.0.1',
            'port_out': self.port2, 'pprefix': ''}
        processes.append(Process(target=twproxy.main, args=(proxy_args,)))          # proxy
        for i in range(N):
            processes.append(Process(target=tcpcl.tcp_cl, args=(dc['client'], ls, ls_out,))) # N clients

        processes[0].start()
        processes[1].start()
        time.sleep(0.1)
        for p in processes[2:]:
            p.start()
            time.sleep(0.1)

        now = time.time()
        while N*len(self.test_msg) != len(ls_out) and \
          (now+2 > time.time()):
            time.sleep(1)
        print("tcp = %.3f" %(time.time() - now))

        processes.reverse()
        for p in processes:
            p.terminate()
            p.join()

        for d, k in zip([c for c in conn], [conn[c] for c in conn]):
            os.system('fuser -k '+conn[d]['port']+'/tcp')

        #print(ls_out)
        print('tcp msg count %i' %len(ls_out))
        self.assertTrue(N*len(self.test_msg) == len(ls_out))

# 需要导入模块: from multiprocessing import Manager [as 别名]
# 或者: from multiprocessing.Manager import list [as 别名]
    def multiupload(self, filename, hosts):
        """Upload file to multiple hosts simultaneously

        The upload will be attempted for each host until the optimal file
        redundancy is achieved (a percentage of successful uploads) or the host
        list is depleted.

        Args:
            filename (str): The filename of the file to upload.
            hosts (list): A list of hosts as defined in the master host list.
        Returns:
            A list of dicts with 'host_name' and 'url' keys for all successful
            uploads or an empty list if all uploads failed.

        """
        manager = Manager()
        successful_uploads = manager.list([])

        def f(host):
            if len(successful_uploads)/float(len(hosts)) < settings.MIN_FILE_REDUNDANCY:
                # Optimal redundancy not achieved, keep going
                result = self.upload_to_host(filename, host)
                if 'error' in result:
                    self._host_errors[host] += 1
                else:
                    successful_uploads.append(result)

        multiprocessing.dummy.Pool(len(hosts)).map(f, self._hosts_by_success(hosts))

        return list(successful_uploads)

# 需要导入模块: from multiprocessing import Manager [as 别名]
# 或者: from multiprocessing.Manager import list [as 别名]
 def run_multiprocesses_likelihood(self):
     lik = 0.0
     workers = []
     workers_no = self.configuration.num_threads
     corpusSplitlist = self.split_average_data(workers_no)
     
     likmanager = Manager()
     ManagerReturn_corpusSplitlist = []
     ManagerReturn_corpusSplitlist_lik = []
     for dataSplit in corpusSplitlist:
         likreturn_dataSplit = likmanager.list()
         likreturn_dataSplit_likvalue = likmanager.Value("",0.0)
         worker = Process(target=self.splitlikelihood, args=(dataSplit, likreturn_dataSplit, likreturn_dataSplit_likvalue))
         worker.start()
         workers.append(worker)
         ManagerReturn_corpusSplitlist.append(likreturn_dataSplit)
         ManagerReturn_corpusSplitlist_lik.append(likreturn_dataSplit_likvalue)
     for w in workers:
         w.join()
     
     # compute all the likelihood for the splits:
     for v in ManagerReturn_corpusSplitlist_lik:
         lik += v.value
     # update all the docs into corpus, since we compute the doc distribution in likelihood()
     self.corpus.clear()
     for dataSplit in ManagerReturn_corpusSplitlist:
         for doc in dataSplit:
             self.corpus.append(doc)
     
     return lik

# 需要导入模块: from multiprocessing import Manager [as 别名]
# 或者: from multiprocessing.Manager import list [as 别名]
    def __init__(self,port):
        manager = Manager()
        self.status=manager.dict()
        self.sendbuf=manager.list()
        self.p = Process(target=SocketManager, args=(port,self.status,self.sendbuf) )
	self.p.daemon=True
        self.p.start()