Python Queue.get_nowait方法代码示例

# 需要导入模块: from multiprocessing import Queue [as 别名]
# 或者: from multiprocessing.Queue import get_nowait [as 别名]
def test_inceptionresnetv2_notop():
    def target(queue):
        model = applications.InceptionResNetV2(weights=None, include_top=False)
        queue.put(model.output_shape)

    global_image_data_format = K.image_data_format()
    queue = Queue()

    K.set_image_data_format('channels_first')
    p = Process(target=target, args=(queue,))
    p.start()
    p.join()
    K.set_image_data_format(global_image_data_format)
    assert not queue.empty(), 'Model creation failed.'
    model_output_shape = queue.get_nowait()
    assert model_output_shape == (None, 1536, None, None)

    K.set_image_data_format('channels_last')
    p = Process(target=target, args=(queue,))
    p.start()
    p.join()
    K.set_image_data_format(global_image_data_format)
    assert not queue.empty(), 'Model creation failed.'
    model_output_shape = queue.get_nowait()
    assert model_output_shape == (None, None, None, 1536)

# 需要导入模块: from multiprocessing import Queue [as 别名]
# 或者: from multiprocessing.Queue import get_nowait [as 别名]
def stress_test ( addr, port, filename, totalcount, processes):
    from multiprocessing import Process, Queue
    import Queue as ThQueue
    qfrom = Queue()
    qto = Queue()
    
    for i in xrange(0,totalcount): qfrom.put ( (i, addr, port, filename) )
    p = [Process(target=stressing_tester, args=(qfrom,qto))  for i in range (0, processes)]
    map (lambda x: x.start(), p)
    map (lambda x: x.join(), p)
    try:
        while True:
            print qto.get_nowait()
    except ThQueue.Empty:
        pass        

# 需要导入模块: from multiprocessing import Queue [as 别名]
# 或者: from multiprocessing.Queue import get_nowait [as 别名]
class _workerQpushTimer():
    def __init__(self):
	self.syncPeriod = 2 
	self.timer = None
	self.Qinit()
    def Qinit(self):
	self.syncTmpQ = Queue()
    # flush remain items in queue, and then close and join_thread
    def Qflush(self):
	while True:
	    try:
		self.syncTmpQ.get(True, comm.FLUSH_TIMEOUT)
	    except Empty:
		break
	self.syncTmpQ.close()
	self.syncTmpQ.join_thread()
    def enableTimer(self, workerPool):
	self.timer = Timer(self.syncPeriod, self.pushToWorkerQ, [workerPool])
	self.timer.start()
    def disableTimer(self):
	if self.timer is not None:
	    self.timer.cancel()
    # function executed periodically, used to sync queue between main process queue and worker queue
    def pushToWorkerQ(self, workerPool):
	while not comm.done.value:
	    try:
		item = self.syncTmpQ.get_nowait() 
		for w in workerPool:
		    w.queue.put_nowait(item)
	    except Empty:
		break
	if not comm.done.value:
	    self.enableTimer(workerPool)

# 需要导入模块: from multiprocessing import Queue [as 别名]
# 或者: from multiprocessing.Queue import get_nowait [as 别名]
    def instantiate(self, stream=None):
        """ Start a local worker process

        Blocks until the process is up and the center is properly informed
        """
        if self.process and self.process.is_alive():
            raise ValueError("Existing process still alive. Please kill first")
        q = Queue()
        self.process = Process(target=run_worker,
                               args=(q, self.ip, self.center.ip,
                                     self.center.port, self.ncores,
                                     self.port, self._given_worker_port,
                                     self.local_dir, self.services, self.name))
        self.process.daemon = True
        self.process.start()
        while True:
            try:
                msg = q.get_nowait()
                if isinstance(msg, Exception):
                    raise msg
                self.worker_port = msg['port']
                assert self.worker_port
                self.worker_dir = msg['dir']
                break
            except queues.Empty:
                yield gen.sleep(0.1)
        logger.info("Nanny %s:%d starts worker process %s:%d",
                    self.ip, self.port, self.ip, self.worker_port)
        q.close()
        raise gen.Return('OK')

# 需要导入模块: from multiprocessing import Queue [as 别名]
# 或者: from multiprocessing.Queue import get_nowait [as 别名]
class QueueEventsSub:

    def __init__(self, maxsize=0):
        self._maxsize = maxsize
        self._q = Queue(maxsize=maxsize)

    def put_event(self, e):
        if self._q.qsize() == self._maxsize and self._maxsize != 0:
            self._q.get_nowait()

        self._q.put(e)

    def get_event(self):
        if self._q.qsize() == 0:
            return None
        return self._q.get()

# 需要导入模块: from multiprocessing import Queue [as 别名]
# 或者: from multiprocessing.Queue import get_nowait [as 别名]
class Quu(): #Singleton
    def __init__(self):
        self.queue = Queue()
        logger.debug('new queue')

    def addFolders(self, aLeftFolder, aRightFolder):
        logger.debug('adding new item in the quu')
        myComparator = FolderComparator(aLeftFolder, aRightFolder)
        myComparator.setSyncType("keepboth")
        self.queue.put(myComparator)

    def getQuu(self):
        return self.queue

    def getNext(self):
        """
        @return FolderComparator the folder
        """
        try:
            return self.queue.get_nowait()
        except BaseException:
            return None

    def getSize(self):
        return self.queue.qsize()

# 需要导入模块: from multiprocessing import Queue [as 别名]
# 或者: from multiprocessing.Queue import get_nowait [as 别名]
    def instantiate(self, stream=None, environment=None):
        """ Start a local worker process

        Blocks until the process is up and the scheduler is properly informed
        """
        if environment:
            if not os.path.isabs(environment):
                environment = os.path.join(self.local_dir, environment)
            self.environment = environment

        with log_errors():
            if self.process and isalive(self.process):
                raise ValueError("Existing process still alive. Please kill first")

            if self.environment != nanny_environment:
                with tmpfile() as fn:
                    self.process = run_worker_subprocess(self.environment, self.ip,
                            self.scheduler.ip, self.scheduler.port, self.ncores,
                            self.port, self._given_worker_port, self.name,
                            self.memory_limit, self.loop, fn, self.quiet)

                    while not os.path.exists(fn):
                        yield gen.sleep(0.01)

                    while True:
                        try:
                            with open(fn) as f:
                                msg = json.load(f)
                            self.worker_port = msg['port']
                            self.worker_dir = msg['local_directory']
                            break
                        except JSONDecodeError:
                            yield gen.sleep(0.01)
            else:
                q = Queue()
                self.process = Process(target=run_worker_fork,
                                       args=(q, self.ip, self.scheduler.ip,
                                             self.scheduler.port, self.ncores,
                                             self.port, self._given_worker_port,
                                             self.local_dir, self.services, self.name,
                                             self.memory_limit))
                self.process.daemon = True
                self.process.start()
                while True:
                    try:
                        msg = q.get_nowait()
                        if isinstance(msg, Exception):
                            raise msg
                        self.worker_port = msg['port']
                        self.worker_dir = msg['dir']
                        assert self.worker_port
                        break
                    except queues.Empty:
                        yield gen.sleep(0.1)



            logger.info("Nanny %s:%d starts worker process %s:%d",
                        self.ip, self.port, self.ip, self.worker_port)
            raise gen.Return('OK')

# 需要导入模块: from multiprocessing import Queue [as 别名]
# 或者: from multiprocessing.Queue import get_nowait [as 别名]
class Webserver():
    def __init__(self):
        TMP_BC = "/tmp/buildchimp_web"
        self.q_process_to_main = Queue()
        self.basepath = TMP_BC
        self.process = None
        self.pngPath = TMP_BC + "/screenshot.png"
        self.serving_string = False

    def __del__(self):
        self.stop()

    def __str__(self):
        if not self.serving_string:
            try:
                self.serving_string = self.q_process_to_main.get_nowait()
            except: pass
        return self.serving_string if self.servingstring else ""

    def start(self):
        self_copy = copy.copy(self)
        self.process = Process(target = serve, args = (self_copy, self.q_process_to_main) )
        self.process.start()

    def stop(self):
        if self.process:
            self.process.terminate()

    def wait_on_msg(self, timeout=10):
        try:
            got = self.q_process_to_main.get(block=True, timeout=timeout)
            return got
        except:
            return None

# 需要导入模块: from multiprocessing import Queue [as 别名]
# 或者: from multiprocessing.Queue import get_nowait [as 别名]
    def execute_action(self, action):
        event = Event()
        queue = Queue()
        proc = Process(
            target=execute_action_proc,
            args=(self.execute, action, event, queue))
        proc.start()

        # Send heartbeat.
        heartbeat_retry = 0
        while not event.is_set():
            event.wait(config.ACTIVITY_HEARTBEAT_INTERVAL)
            try:
                res = self.heartbeat(self.task_token)
                if res['cancelRequested']:
                    proc.terminate()
                    proc.join()
                    return Result('cancelled', -1, '', '', '', -1)
            except Exception as err:
                if heartbeat_retry <= config.ACTIVITY_HEARTBEAT_MAX_RETRY:
                    heartbeat_retry += 1
                    continue
                else:
                    proc.terminate()
                    proc.join()
                    raise

        # Evaluate the result.
        result = queue.get_nowait()
        proc.join()
        return result

# 需要导入模块: from multiprocessing import Queue [as 别名]
# 或者: from multiprocessing.Queue import get_nowait [as 别名]
def _get_output_shape(model_fn):
    if K.backend() == 'cntk':
        # Create model in a subprocess so that
        # the memory consumed by InceptionResNetV2 will be
        # released back to the system after this test
        # (to deal with OOM error on CNTK backend).
        # TODO: remove the use of multiprocessing from these tests
        # once a memory clearing mechanism
        # is implemented in the CNTK backend.
        def target(queue):
            model = model_fn()
            queue.put(model.output_shape)
        queue = Queue()
        p = Process(target=target, args=(queue,))
        p.start()
        p.join()
        # The error in a subprocess won't propagate
        # to the main process, so we check if the model
        # is successfully created by checking if the output shape
        # has been put into the queue
        assert not queue.empty(), 'Model creation failed.'
        return queue.get_nowait()
    else:
        model = model_fn()
        return model.output_shape

# 需要导入模块: from multiprocessing import Queue [as 别名]
# 或者: from multiprocessing.Queue import get_nowait [as 别名]
def f(idx, q,r):
    path = "data%s"%(idx)
    os.makedirs(path)
    while True:
        item = q.get()
        if( item.item_type == ITEM_QUIT ):
            break;

        count = 0
        localQueue = Queue()
        current = item.data
        while True:
            print current
            fo = urlopen(current)
            data = fo.read()
            name = "%s/%s"%(path,count)
            fw = open( name, "w" )
            count = count + 1
            fw.write(data)
            fw.close()
            fo.close()
            p = MyHTMLParser()
            try:
                p.feed(data)
            except:
                pass

            for href in p.hrefs:
                print item.data, ": ", href

            try:
                current = localQueue.get_nowait()
            except:
                break;

# 需要导入模块: from multiprocessing import Queue [as 别名]
# 或者: from multiprocessing.Queue import get_nowait [as 别名]
	def Launch(self):
		lastcheck = 0
		# process for checking for levels to be saved
		p = None
		# queue to communicate with level saving process
		q = Queue()
		q.put("START", block=True)
		
		# just keep doing this forever
		while True:
			self.Pump()
			# periodically check for unsaved levels with changes and save them
			if lastcheck < time() - settings.SAVEINTERVAL and not q.empty():
				# check if the running process has sent back a list of levels it has saved
				try:
					saved = q.get_nowait()
				except Empty:
					# in rare cases, q.empty() might have returned the wrong answer
					saved = None
				
				# incase q.empty() returned the wrong answer
				if not saved is None:
					# if we actually saved some levels
					if saved != "START":
						# write a log about it
						[self.Log("SAVED %s: %s" % s) for s in saved]
						# update our last saved array
						self.SetSaved([s[1] for s in saved if s[0] == "LEVEL"])
					# launch process to save all unsaved levels
					p = Process(target=SaveData, args=(q, self.levels, self.clientData)).start()
					# update last checked time
					lastcheck = time()
			# make sure we don't eat 100% of CPU
			sleep(0.0001)

# 需要导入模块: from multiprocessing import Queue [as 别名]
# 或者: from multiprocessing.Queue import get_nowait [as 别名]
class PPool(object):
    def __init__(self, func, single=False):
        self.func = func
        self.processes = []
        self.single = single
        self.queue = Queue()

    def submit(self, *args, **kwargs):
        kwargs["queue"] = self.queue
        p = Process(target=self.func, args=args, kwargs=kwargs)
        self.processes.append(p)
        p.start()
        if self.single:
            p.join()

    def __enter__(self):
        return self

    def __exit__(self, x, y, z):
        for p in self.processes:
            p.join()
        log = ''
        while not self.queue.empty():
            l = self.queue.get_nowait()
            log += l
            logging.error(l)
        if "Critical error!" in log:
            exit(1)

# 需要导入模块: from multiprocessing import Queue [as 别名]
# 或者: from multiprocessing.Queue import get_nowait [as 别名]
class RepeatPool:

    """Implements repeating several times the same function through processes
    and returning the list of results. Takes care of using a different numpy
    random state in each process.
    """

    def __init__(self, target):
        self.n_processes = cpu_count()
        self.target = target

    def work(self, seed):
        np.random.seed(seed)
        result = self.target()
        self.result_queue.put(result)

    def run(self, n):
        self.workers = [None] * self.n_processes
        self.to_go = n  # Number of repetitions still to be run
        self.result_queue = Queue()
        results = []
        while self._still_working():
            self._clean()
            self._fill()
            try:  # Waits for at least one job to push results.
                results.append(self.result_queue.get(False, 1))
            except Empty:  # The timeout prevents blocking if jobs fail to push
                pass
        # Collect remaining results just in case
        results.extend([self.result_queue.get_nowait()
                        for _ in range(n - len(results))])
        return results

    def _still_working(self):
        return self.to_go > 0 or any([w is not None for w in self.workers])

    def _start_or_None(self):
        if self.to_go > 0:
            self.to_go -= 1
            p = Process(target=self.work, args=(np.random.randint(2**32),))
            p.start()
            return p
        else:
            return None

    def _clean(self):
        self.workers = [w if w is None or w.is_alive() else self._clean_job(w)
                        for w in self.workers]

    def _clean_job(self, w):
        if w.exitcode > 0:
            raise RuntimeError("A subprocess has failed.")
        w.join()
        return None

    def _fill(self):
        self.workers = [self._start_or_None()
                        if w is None else w for w in self.workers]

# 需要导入模块: from multiprocessing import Queue [as 别名]
# 或者: from multiprocessing.Queue import get_nowait [as 别名]
def test_inceptionresnetv2_variable_input_channels():
    def target(queue, input_shape):
        model = applications.InceptionResNetV2(weights=None, include_top=False, input_shape=input_shape)
        queue.put(model.output_shape)

    queue = Queue()
    p = Process(target=target, args=(queue, (None, None, 1)))
    p.start()
    p.join()
    assert not queue.empty(), 'Model creation failed.'
    model_output_shape = queue.get_nowait()
    assert model_output_shape == (None, None, None, 1536)

    p = Process(target=target, args=(queue, (None, None, 4)))
    p.start()
    p.join()
    assert not queue.empty(), 'Model creation failed.'
    model_output_shape = queue.get_nowait()
    assert model_output_shape == (None, None, None, 1536)