Python MPI.Status方法代码示例

# 需要导入模块: from mpi4py import MPI [as 别名]
# 或者: from mpi4py.MPI import Status [as 别名]
def slave_set(self):
        if self.rank > 0:
#            print("SLAVE : ", self.rank, " starting...")
            status = MPI.Status()
#            print("SLAVE : ", self.rank, " probing for message...")
            msg = self.COMM.Probe(0, MPI.ANY_TAG, status=status)
#            print("SLAVE : ", self.rank, " recieved a message... ", status.Get_tag())
            self.working = True
            if status.Get_tag() == tags.WORK:
                workingBlock = self.COMM.recv(source=0, tag=tags.WORK)
#                print("SLAVE : ", self.rank, " just recieved ", workingBlock)
                self.curr_block = workingBlock
                self.working = True
                return True, workingBlock
            else:
                self.working = False
                workingBlock = self.COMM.recv(source=0, tag=tags.KILL)
#                print("SLAVE : ", self.rank, " dying...")
                return False, 0
        else:
            return False, 0 

# 需要导入模块: from mpi4py import MPI [as 别名]
# 或者: from mpi4py.MPI import Status [as 别名]
def _dispatch_loop(self):
        comm = self.comm        

        while True:

            # Dispatch as many tasks as possible before checking for shutdown
            while self.task_dest:
                try:
                    task = self.task_queue.popleft()
                    task_dest = self.task_dest.popleft()
                except IndexError:
                    break
                else:
                    comm.send(task, dest = task_dest, tag = self.task_tag )

            status = MPI.Status()
            comm.Iprobe(self.master_rank, self.announce_tag, status)
            message_tag = status.Get_tag()

            # Check for announcements
            if message_tag == self.announce_tag:
                messages = comm.recv(source = self.master_rank, tag = self.announce_tag)
                if 'shutdown' in messages:
                    log.debug('exiting _dispatch_loop()')
                    return 

# 需要导入模块: from mpi4py import MPI [as 别名]
# 或者: from mpi4py.MPI import Status [as 别名]
def _read(self):
        result = super(MPIWrapper, self)._read()
        if result is not None:
            return result

        status = MPI.Status()
        msg = comm.recv(source=MPI.ANY_SOURCE, tag=MPI.ANY_TAG, status=status)
        tag = status.Get_tag()
        while tag == STATUS_TERMINATED:
            self.terminated += 1
            if self.terminated >= self._num_sources:
                break
            else:
                msg = comm.recv(source=MPI.ANY_SOURCE,
                                tag=MPI.ANY_TAG,
                                status=status)
                tag = status.Get_tag()
        return msg, tag 

# 需要导入模块: from mpi4py import MPI [as 别名]
# 或者: from mpi4py.MPI import Status [as 别名]
def _write(self, name, data):
        try:
            targets = self.targets[name]
        except KeyError:
            # no targets
            # self.pe.log('Produced output: %s' % {name: data})
            return
        for (inputName, communication) in targets:
            output = {inputName: data}
            dest = communication.getDestination(output)
            for i in dest:
                try:
                    # self.pe.log('Sending %s to %s' % (output, i))
                    request = comm.isend(output, tag=STATUS_ACTIVE, dest=i)
                    status = MPI.Status()
                    request.Wait(status)
                except:
                    self.pe.log(
                        'Failed to send data stream "%s" to rank %s: %s'
                        % (name, i, traceback.format_exc())) 

# 需要导入模块: from mpi4py import MPI [as 别名]
# 或者: from mpi4py.MPI import Status [as 别名]
def mpi_master_spin_tasks(task, gdirs):
    comm = OGGM_MPI_COMM
    cfg_store = cfg.pack_config()
    msg_list = ([gdir for gdir in gdirs if gdir is not None] +
                [None] * OGGM_MPI_SIZE)

    _imprint("Starting MPI task distribution...")

    comm.bcast((cfg_store, task), root=OGGM_MPI_ROOT)

    status = MPI.Status()
    for msg in msg_list:
        comm.recv(source=MPI.ANY_SOURCE, status=status)
        comm.send(obj=msg, dest=status.Get_source())

    _imprint("MPI task distribution done, collecting results...")

    comm.gather(sendobj=None, root=OGGM_MPI_ROOT)

    _imprint("MPI task results gotten!") 

# 需要导入模块: from mpi4py import MPI [as 别名]
# 或者: from mpi4py.MPI import Status [as 别名]
def __init__(self, cpus_per_task, comm=None, debug=False, use_all_cpus=False):

        if debug:
            self.logger.setLevel(logging.DEBUG)

        self.cpus_per_task = cpus_per_task
        self.use_all_cpus  = use_all_cpus

        # the base communicator
        self.basecomm = MPI.COMM_WORLD if comm is None else comm
        self.rank     = self.basecomm.rank
        self.size     = self.basecomm.size

        # need at least one
        if self.size == 1:
            raise ValueError("need at least two processes to use a TaskManager")

        # communication tags
        self.tags = enum('READY', 'DONE', 'EXIT', 'START')

        # the task communicator
        self.comm = None

        # store a MPI status
        self.status = MPI.Status() 

# 需要导入模块: from mpi4py import MPI [as 别名]
# 或者: from mpi4py.MPI import Status [as 别名]
def pt_sample():
    # Define MPI message tags
    tags = enum('READY', 'DONE', 'EXIT', 'START')

    # Initializations and preliminaries
    comm = MPI.COMM_WORLD   # get MPI communicator object
    size = comm.size        # total number of processes
    rank = comm.rank        # rank of this process
    status = MPI.Status()   # get MPI status object

    if rank == 0:
        print('Here')
        master_process(comm, size, tags, status)
    else:
        print('worker')
        worker_process(comm, rank, tags, status)

    print('Done!') 

# 需要导入模块: from mpi4py import MPI [as 别名]
# 或者: from mpi4py.MPI import Status [as 别名]
def __init__(self, rank=None, nonMPIMode=False, logger=None):
        if nonMPIMode:
            self.comm = None
            self.stat = None
            self.nRank = 0
            self.totalRanks = 1
            self.selectSource = None
        else:
            from mpi4py import MPI
            self.comm = MPI.COMM_WORLD
            self.stat = MPI.Status()
            self.nRank = self.comm.Get_rank()
            self.totalRanks = self.comm.Get_size()
            self.selectSource = MPI.ANY_SOURCE

        self.logger = logger

        # for message in rank 0
        self.hasMessage = False
        self.recvQueue = recvQueue
        self.sendQueue = sendQueue


    # get rank of itself 

# 需要导入模块: from mpi4py import MPI [as 别名]
# 或者: from mpi4py.MPI import Status [as 别名]
def __init__(self, comm, **kwargs):
        super(NN_Trainer, self).__init__()
        self.comm = comm   # get MPI communicator object
        self.world_size = comm.Get_size() # total number of processes
        self.rank = comm.Get_rank() # rank of this Worker
        #self.status = MPI.Status()
        self.cur_step = 0
        self.next_step = 0 # we will fetch this one from parameter server

        self.batch_size = kwargs['batch_size']
        self.max_epochs = kwargs['max_epochs']
        self.momentum = kwargs['momentum']
        self.lr = kwargs['learning_rate']
        self._max_steps = kwargs['max_steps']
        self.network_config = kwargs['network']
        self.comm_type = kwargs['comm_method']
        self.kill_threshold = kwargs['kill_threshold']
        self._eval_batch_size = 100
        self._eval_freq = kwargs['eval_freq']
        self._train_dir = kwargs['train_dir']
        self._compress_grad = kwargs['compress_grad']
        self._device = kwargs['device']

        # this one is going to be used to avoid fetch the weights for multiple times
        self._layer_cur_step = [] 

# 需要导入模块: from mpi4py import MPI [as 别名]
# 或者: from mpi4py.MPI import Status [as 别名]
def worker():
    from mpi4py import MPI
    while True:
        status = MPI.Status()
        ret = MPI.COMM_WORLD.recv(source=0, tag=MPI.ANY_TAG, status=status) 

        if status.tag == 10:
            # Workload received
            func = ret['func']
            if ret.get('unpack'):
                res = func(*ret['input_data'])
            else:
                res = func(ret['input_data'])

            # Done, let's send it back
            MPI.COMM_WORLD.send(dict(job_index=ret['job_index'], output_data=res), dest=0, tag=2)

        elif status.tag == 666:
            # Kill code
            sys.exit(0) 

# 需要导入模块: from mpi4py import MPI [as 别名]
# 或者: from mpi4py.MPI import Status [as 别名]
def _handle_message(self):
        """Handle received messages.

        Receive record update messages of the form
            ('action', record_id, params)
        where 'action' is the name of an EvalRecord method and params is
        the list of parameters.  The record_id should be recorded in the
        hub's records table (which happens whenever it is referenced in
        a message sent to a worker).

        On a message indicating that the worker is done with the record,
        we add the worker that sent the message back to the free pool.
        """
        logger.debug("Handle incoming message")
        s = MPI.Status()
        data = comm.recv(status=s, source=MPI.ANY_SOURCE, tag=0)
        logger.debug("Received message: %s", data)
        mname = data[0]
        record = self._recids[data[1]]
        method = getattr(record, mname)
        method(*data[2:])
        if mname == 'complete' or mname == 'cancel' or mname == 'kill':
            logger.debug("Re-queueing worker")
            self._workers.append(s.source)
        self.ping() 

# 需要导入模块: from mpi4py import MPI [as 别名]
# 或者: from mpi4py.MPI import Status [as 别名]
def _receive_loop(self):
        comm = self.comm 

        while True:

            status = MPI.Status()
            comm.Iprobe(MPI.ANY_SOURCE, MPI.ANY_TAG, status)
            message_src = status.Get_source()
            message_tag = status.Get_tag()

            # results are tuples of (task_id, {'result', 'exception'}, value)
            if message_tag == self.result_tag:
                (task_id, result_stat, result_value) = comm.recv(source = message_src, tag = message_tag)

                ft = self.pending_futures.pop(task_id)

                if result_stat == 'exception':
                    ft._set_exception(*result_value)
# Check with Matt on what else to do for an exception
                else:
                    ft._set_result(result_value)
                    self.task_dest.append(message_src)

            # Check for announcements
            elif message_tag == self.announce_tag:
                messages = comm.recv(source = message_src, tag = message_tag)
                if 'shutdown' in messages:
                    log.debug('exiting _receive_loop()')
                    return 

# 需要导入模块: from mpi4py import MPI [as 别名]
# 或者: from mpi4py.MPI import Status [as 别名]
def _create_worker(self):
        comm = self.comm

        while True:

            status = MPI.Status()
            comm.Probe(self.master_rank, MPI.ANY_TAG, status)
            message_src = self.master_rank
            message_tag = status.Get_tag()

            # Check for available task 
            if message_tag == self.task_tag:

                task = comm.recv(source = message_src, tag = message_tag)

                try:
                    result_value = task.fn(*task.args, **task.kwargs)
                except Exception as e:
                    result_object = (task.task_id, 'exception', result_value)
                else:
                    result_object = (task.task_id, 'result', result_value)

                comm.send(result_object, dest = self.master_rank, tag = self.result_tag)

            # Check for announcements
            if message_tag == self.announce_tag:
                messages = comm.recv(source = message_src, tag = message_tag)
                if 'shutdown' in messages:
                    return 

# 需要导入模块: from mpi4py import MPI [as 别名]
# 或者: from mpi4py.MPI import Status [as 别名]
def receive(wrapper):
    while wrapper.terminated < wrapper._num_sources:
        status = MPI.Status()
        msg = comm.recv(source=MPI.ANY_SOURCE, tag=MPI.ANY_TAG, status=status)
        tag = status.Get_tag()
        # print('Received %s, %s' % (msg, tag))
        if tag == STATUS_TERMINATED:
            wrapper.terminated += 1
        else:
            wrapper.input_data.put((msg, tag))
        # self.wrapper.pe.log('Queue size: %s'%self.wrapper.input_data.qsize())
    # put the final terminate block into the queue
    wrapper.input_data.put((None, STATUS_TERMINATED)) 

# 需要导入模块: from mpi4py import MPI [as 别名]
# 或者: from mpi4py.MPI import Status [as 别名]
def _write(self, name, data):
        try:
            targets = self.targets[name]
        except KeyError:
            # no targets
            # self.pe.log('Produced output: %s' % {name: data})
            return
        for (inputName, communication) in targets:
            output = {inputName: data}
            dest = communication.getDestination(output)
            for i in dest:
                # self.pe.log('Sending %s to %s' % (output, i))
                request = comm.isend(output, tag=STATUS_ACTIVE, dest=i)
                status = MPI.Status()
                request.Wait(status)