Python queues.SimpleQueue类代码示例

def export_table(host, port, auth_key, db, table, directory, fields, format, error_queue, progress_info, stream_semaphore, exit_event):
    writer = None

    try:
        # This will open at least one connection for each rdb_call_wrapper, which is
        # a little wasteful, but shouldn't be a big performance hit
        conn_fn = lambda: r.connect(host, port, auth_key=auth_key)
        rdb_call_wrapper(conn_fn, "count", get_table_size, db, table, progress_info)
        table_info = rdb_call_wrapper(conn_fn, "info", write_table_metadata, db, table, directory)

        with stream_semaphore:
            task_queue = SimpleQueue()
            writer = launch_writer(format, directory, db, table, fields, task_queue, error_queue)
            writer.start()

            rdb_call_wrapper(conn_fn, "table scan", read_table_into_queue, db, table,
                             table_info["primary_key"], task_queue, progress_info, exit_event)
    except (r.RqlError, r.RqlDriverError) as ex:
        error_queue.put((RuntimeError, RuntimeError(ex.message), traceback.extract_tb(sys.exc_info()[2])))
    except:
        ex_type, ex_class, tb = sys.exc_info()
        error_queue.put((ex_type, ex_class, traceback.extract_tb(tb)))
    finally:
        if writer is not None and writer.is_alive():
            task_queue.put(("exit", "event")) # Exit is triggered by sending a message with two objects
            writer.join()
        else:
            error_queue.put((RuntimeError, RuntimeError("writer unexpectedly stopped"),
                             traceback.extract_tb(sys.exc_info()[2])))

class Logger(object):
    def __init__(self, logfilepath):
        try:
            os.remove(logfilepath)
        except OSError:
            pass
        
        self.logfilepath = logfilepath
        self.logq = SimpleQueue()
        
        self.tags = ''
        self.num_tags = 0
        
    def add_tag(self, tag):
        #self.log("adding tag {}".format(tag))
        self.num_tags += 1
        
        if self.tags != '':
            self.tags = self.tags + '.' + tag
        else:
            self.tags = tag
            
    def remove_tag(self):
        #self.log("removing tag")
        tags = self.tags.split('.')
        self.tags = ".".join(tags[:-1])
        self.num_tags -= 1
        
    def get_tag_part(self):
        if self.tags != '':
            return self.tags + ": "
        else:
            return ''
        
    def log(self, message, start_group=None, end_group=None):
        assert(type(message)==str)        
        self.logq.put(" "*self.num_tags*4 + self.get_tag_part() + message + '\n')
            
    def getlog(self):
        return self.logq.get()
            
    def getlogs(self, n=None):
        logs = []
        if n == None:
            while not self.logq.empty():
                logs.append(self.getlog())
        else:
            assert(type(n)==int)
            while not (self.logq.empty() or len(logs) == n):
                logs.append(self.getlog())
                
        return logs
        
    def write_to_file(self):        
        # mode 'a' for append
        with open(self.logfilepath, 'a') as f:
            f.writelines(self.getlogs())

    def test_can_pickle_via_queue(self):
        """
        https://github.com/andresriancho/w3af/issues/8748
        """
        sq = SimpleQueue()
        u1 = URL('http://www.w3af.com/')
        sq.put(u1)
        u2 = sq.get()

        self.assertEqual(u1, u2)

def QuiverPlotter(num):
	data_q = SimpleQueue()

	plot = Process(target=quiverPlotter,args=(data_q,num))
	plot.start()

	try:
		while True:
			data = (yield)
			if data_q.empty() == False:
				continue
			data_q.put(data)
	except GeneratorExit:
		plot.join()

def Plotter3D(plots,scale):
	data_q = SimpleQueue()

	plot = Process(target=plotter3D,args=(data_q,plots,scale))
	plot.start()

	data = {}
	try:
		while True:
			data.update((yield))
			if data_q.empty() == False:
				continue
			data_q.put(data)
	except GeneratorExit:
		pass

    def __init__(self, key, task_group, randomize):
        self.key = key
        self.gen_worker = task_group['gen_worker']
        self.task_ids = task_group['task_ids']
        self.is_parallel = task_group['is_parallel']
        if self.is_parallel:
            self.randomize = randomize
            if self.randomize:
                random.shuffle(self.task_ids)
        else:
            self.randomize = False
        self.result_queue = SimpleQueue()
        self.task_queue = SimpleQueue()

        # Don't expose queues file descriptors over Popen to, say, tarantool
        # running tests.
        set_fd_cloexec(self.result_queue._reader.fileno())
        set_fd_cloexec(self.result_queue._writer.fileno())
        set_fd_cloexec(self.task_queue._reader.fileno())
        set_fd_cloexec(self.task_queue._writer.fileno())

        for task_id in self.task_ids:
            self.task_queue.put(task_id)
        self.worker_ids = set()
        self.done = False
        self.done_task_ids = set()

class StatusTracker(object):
    def __init__(self):
        self.logq = SimpleQueue()
        self.history = []
        
    def put(self, msg):
        assert(type(msg)==str)
        self.logq.put(msg)
        
    def flushq(self):
        while not self.logq.empty():
            self.history.append(self.logq.get())
        self.prune_history()
            
    def prune_history(self):
        self.history = self.history[-100:]

    def __init__(self, data_structure, processes, scan_function, init_args, _mp_init_function):
        """ Init the scanner.

        data_structure is a world.DataSet
        processes is the number of child processes to use
        scan_function is the function to use for scanning
        init_args are the arguments passed to the init function
        _mp_init_function is the function used to init the child processes
        """
        assert (isinstance(data_structure, world.DataSet))
        self.data_structure = data_structure
        self.list_files_to_scan = data_structure._get_list()
        self.processes = processes
        self.scan_function = scan_function

        # Queue used by processes to pass results
        self.queue = SimpleQueue()
        init_args.update({'queue': self.queue})
        # NOTE TO SELF: initargs doesn't handle kwargs, only args!
        # Pass a dict with all the args
        self.pool = multiprocessing.Pool(processes=processes, initializer=_mp_init_function, initargs=(init_args,))

        # TODO: make this automatic amount
        # Recommended time to sleep between polls for results
        self.SCAN_START_SLEEP_TIME = 0.001
        self.SCAN_MIN_SLEEP_TIME = 1e-6
        self.SCAN_MAX_SLEEP_TIME = 0.1
        self.scan_sleep_time = self.SCAN_START_SLEEP_TIME
        self.queries_without_results = 0
        self.last_time = time()
        self.MIN_QUERY_NUM = 1
        self.MAX_QUERY_NUM = 5

        # Holds a friendly string with the name of the last file scanned
        self._str_last_scanned = None

def launch_graph_plot():
    q = SimpleQueue()
    Pyro4.config.HOST="10.1.1.2"
    daemon = Pyro4.Daemon()
    ns = Pyro4.locateNS()
    p = Process(target=_launch_daemon, args=(daemon, q,))
    p.start()
    graph_plot = GraphPlotPanel()
    while True:
        if not q.empty():
            item = q.get()
            if item[0] == 'time':
                print "got queue:", item
                graph_plot.set_time(item[1])
            elif item[0] == 'vertex_color':
                pass
        graph_plot.run()
        fpsClock.tick(60)

def DensityPlotter(num,size):
	# num = size/scale
	range = [[-size,size],[-size,size]]

	data_q = SimpleQueue()

	plot = Process(target=imagedraw,args=(data_q,num))
	plot.start()

	while True:
		x = (yield)

		if data_q.empty() == False:
			continue

		hist,_,_ = np.histogram2d(x[:,0],x[:,1],bins=num,range=range)
		avg = np.average(hist)
		hist = (hist - avg)/avg
		data_q.put(hist.astype(np.float32))

class RangerControlServer(HTTPServer):
    def __init__(self, fm):
        self.fm = fm
        self.queue = SimpleQueue()
        self.goDie = False
        HTTPServer.__init__(self, ("127.0.0.1", 5964), RangerControlHandler)

    def start(self):
        self.thread = threading.Thread(target=self.process)
        self.thread.start()

    def stop(self):
        self.shutdown()

    def process(self):
        self.serve_forever()

    def check_messages(self):
        if self.queue.empty():
            return None
        return self.queue.get()

    def act_on_messages(self):
        msg = self.check_messages()
        if msg == None:
            return False

        action, arg = msg
        match = re.match(r"/cdtab-(\S+)", action)
        if match != None:
            tab = match.group(1)
            if not (tab in self.fm.tabs):
                self.fm.tab_open(tab, arg)
            else:
                self.fm.tabs[tab].enter_dir(arg)
        elif action == "/cd":
            self.fm.enter_dir(arg)
        elif action == "/cdfirst":
            first_tab = self.fm._get_tab_list()[0]
            self.fm.tabs[first_tab].enter_dir(arg)
        else:
            self.fm.notify("Unknown server command", bad=True)
        return True

 def __init__(self, logfilepath):
     try:
         os.remove(logfilepath)
     except OSError:
         pass
     
     self.logfilepath = logfilepath
     self.logq = SimpleQueue()
     
     self.tags = ''
     self.num_tags = 0

class LinePlotter:
	def __init__(self,*args,**kwargs):
		self.data_q = SimpleQueue()
		self.data = {}

		self.plot = LinePlotterProcess(self.data_q)
		self.plot.add_plot(*args,**kwargs)

	def show(self):
		self.plot.start()

	def add_plot(self,*args,**kwargs):
		self.plot.add_plot(*args,**kwargs)

	def send(self,data):
		if data == GeneratorExit:
			self.plot.join()

		self.data.update(data)
		if self.data_q.empty() != False:
			self.data_q.put(data)

class ErrorMonitor:
    def __init__(self):
        self.pipe = SimpleQueue()
        self.message = None

    def main(self):
        while True:
            message = self.pipe.get()
            if message != 'Q':
                self.message = message[1:]
                LongJump.longjump()
                break
            else:
                self.pipe = None
                break
                    
    def haserror(self):
        """ master only """
        return self.message is not None
    def start(self):
        """ master only """
        self.thread = Thread(target=self.main)
        self.thread.daemon = True
        self.thread.start()
    def join(self):
        """ master only """
        try:
            self.pipe.put('Q')
            self.thread.join()
        except:
            pass
        finally:
            self.thread = None

    def slaveraise(self, type, error, traceback):
        """ slave only """
        message = 'E' * 1 + pickle.dumps((type,
            ''.join(tb.format_exception(type, error, traceback))))
        if self.pipe is not None:
            self.pipe.put(message)

    def setup(self):
        self.pid = os.getpid()
        self.worker_nums = self.config['workers']
        self.worker_class = SyncWorker
        self.queue = SimpleQueue()
        self.setup_logger()
        self.setup_signals()

        addresses = self.config['binds']
        self.sockets = create_sockets(addresses, self.logger)

        addresses_str = ', '.join(map(format_addr_str, addresses))
        self.logger.info('Arbiter booted')
        self.logger.info('Listening on: %s (%s)', addresses_str, self.pid)
        self.logger.info('Using worker: %s', self.worker_class)