Python eventlet.Queue类代码示例

class Dispatcher(EventThread):
    def __init__(self):
        self.proposer_queue = Queue()
        self.acceptor_queue = Queue()
    def _run(self):
        while True:
            #get recved string
            u = do_recv()
            m = Message()
            if not m.depack(u):
                log.debug('depack %s error', u)
                continue
            #check timestamp
            if m.timestamp - get_utc_time() > conf.proposer_timeout:
                log.debug('receive expired package, drop it')
                continue
            #check version
            if m.version != VERSION:
                log.debug('version not match')
                continue
            #check signature
            if m.signature != m.build_hmac_signature():
                log.info('message signature failed, may changed in network')
                continue

            if m.method in (PREPARE, ACCEPT, LEARN, RENEW):
                self.acceptor_queue.put(m)
            elif m.method in (NACK, ACK, OUTDATE):
                self.proposer_queue.put(m)

    def test_blocks_on_pool(self):
        waiter = Queue(0)
        def greedy():
            self.pool.get()
            self.pool.get()
            self.pool.get()
            self.pool.get()
            # No one should be waiting yet.
            self.assertEquals(self.pool.waiting(), 0)
            # The call to the next get will unschedule this routine.
            self.pool.get()
            # So this put should never be called.
            waiter.put('Failed!')

        killable = eventlet.spawn(greedy)

        # no one should be waiting yet.
        self.assertEquals(self.pool.waiting(), 0)

        ## Wait for greedy
        eventlet.sleep(0)

        ## Greedy should be blocking on the last get
        self.assertEquals(self.pool.waiting(), 1)

        ## Send will never be called, so balance should be 0.
        self.assertFalse(not waiter.full())

        eventlet.kill(killable)

    def test_connection_pooling(self):
        with patch('swift.common.memcached.socket') as mock_module:
            # patch socket, stub socket.socket, mock sock
            mock_sock = mock_module.socket.return_value

            # track clients waiting for connections
            connected = []
            connections = Queue()
            errors = []

            def wait_connect(addr):
                connected.append(addr)
                sleep(0.1)  # yield
                val = connections.get()
                if val is not None:
                    errors.append(val)

            mock_sock.connect = wait_connect

            memcache_client = memcached.MemcacheRing(['1.2.3.4:11211'],
                                                     connect_timeout=10)
            # sanity
            self.assertEquals(1, len(memcache_client._client_cache))
            for server, pool in memcache_client._client_cache.items():
                self.assertEqual(2, pool.max_size)

            # make 10 requests "at the same time"
            p = GreenPool()
            for i in range(10):
                p.spawn(memcache_client.set, 'key', 'value')
            for i in range(3):
                sleep(0.1)
                self.assertEqual(2, len(connected))

            # give out a connection
            connections.put(None)

            # at this point, only one connection should have actually been
            # created, the other is in the creation step, and the rest of the
            # clients are not attempting to connect. we let this play out a
            # bit to verify.
            for i in range(3):
                sleep(0.1)
                self.assertEqual(2, len(connected))

            # finish up, this allows the final connection to be created, so
            # that all the other clients can use the two existing connections
            # and no others will be created.
            connections.put(None)
            connections.put('nono')
            self.assertEqual(2, len(connected))
            p.waitall()
            self.assertEqual(2, len(connected))
            self.assertEqual(0, len(errors),
                             "A client was allowed a third connection")
            connections.get_nowait()
            self.assertTrue(connections.empty())

 def __init__ (self, server_url, login_params):
     self.running = True
     self.agent = True
     self.cmd_out_queue = []
     self.cmd_in_queue = []
     self.out_queue = Queue()
     self.in_queue = Queue()
     self.server_url = server_url
     self.login_params = login_params
     Thread.__init__(self)

 def __init__(self):
     self.running = True
     self.configuration = OrderedSet()
     
     self.internalQueue = Queue()
     self.externalQueue = Queue()
     
     self.statesToInvoke = OrderedSet()
     self.historyValue = {}
     self.dm = None
     self.invokeId = None
     self.parentId = None
     self.logger = None

class WSTestGenerator(WebSocketView):

    def handle_websocket(self, ws):
        self._ws = ws
        return super(RangeWebsocket, self)

    def handler(self, ws):
        self.queue = Queue()
        while True:
            m = ws.wait()
#            import ipdb; ipdb.set_trace()
            if m is None:
                break
            self.queue.put(m)

    def __init__(self):
        self.active = False

        self.prefetch_queue = Queue(settings.prefetch['queue_size'])
        self.prefetch_thread = spawn(self.prefetch_worker)
        self.prefetch_thread.link(reraise_errors, greenthread.getcurrent())

        self.given_items = Cache()

        self.postreport_queue = Queue(settings.postreport['queue_size'])
        self.postreport_thread = spawn(self.postreport_worker)
        self.postreport_thread.link(reraise_errors, greenthread.getcurrent())

        self.storage_connections = eventlet.pools.Pool(max_size=settings.storage['max_connections'])
        self.storage_connections.create = StorageConnection

    def handler(self, ws):
        self.queue = Queue()
        while True:
            m = ws.wait()
#            import ipdb; ipdb.set_trace()
            if m is None:
                break
            self.queue.put(m)

class GreenletsThread(Thread):
    """
    Main thread for the program. If running stand alone this will be running
    as a greenlet instead.
    """
    def __init__ (self, server_url, login_params):
        self.running = True
        self.agent = True
        self.cmd_out_queue = []
        self.cmd_in_queue = []
        self.out_queue = Queue()
        self.in_queue = Queue()
        self.server_url = server_url
        self.login_params = login_params
        Thread.__init__(self)

    def apply_position(self, obj_uuid, pos, rot=None):
        cmd = ['pos', obj_uuid, pos, rot]
        self.addCmd(cmd)

    def __getattr__(self, name):
        return ProxyFunction(name, self)

    def apply_scale(self, obj_uuid, scale):
        cmd = ['scale', obj_uuid, scale]
        self.addCmd(cmd)

    def run(self):
        agent = AgentManager(self.in_queue,
                   self.out_queue)
        error = agent.login(self.server_url, self.login_params)
        if error:
            self.out_queue.put(["error", str(error)])
            self.out_queue.put(["agentquit", str(error)])
            while self.out_queue.qsize():
                api.sleep(0.1)
        agent.logger.debug("Quitting")
        self.agent = agent
        self.running = False

    def addCmd(self, cmd):
        self.in_queue.put(cmd)

    def getQueue(self):
        out_queue = []
        while self.out_queue.qsize():
            out_queue.append(self.out_queue.get())
        return out_queue

 def __init__(self, init_urls, max_workers=200):
     self.init_urls = init_urls
     self.max_workers = max_workers
     self.q = Queue()
     self.urls = set()
     self.s = requests.Session()
     self.root_hosts = set()
     for url in init_urls:
         self.q.put(url)
         self.urls.add(url)
         self.root_hosts.add(get_netloc(url))

    def test_putting_to_queue(self):
        timer = eventlet.Timeout(0.1)
        try:
            size = 2
            self.pool = IntPool(min_size=0, max_size=size)
            queue = Queue()
            results = []
            def just_put(pool_item, index):
                self.pool.put(pool_item)
                queue.put(index)
            for index in xrange(size + 1):
                pool_item = self.pool.get()
                eventlet.spawn(just_put, pool_item, index)

            for _ in range(size+1):
                x = queue.get()
                results.append(x)
            self.assertEqual(sorted(results), range(size + 1))
        finally:
            timer.cancel()

    def test_connection_pooling(self):
        with patch('swift.common.memcached.socket') as mock_module:
            # patch socket, stub socket.socket, mock sock
            mock_sock = mock_module.socket.return_value

            # track clients waiting for connections
            connected = []
            connections = Queue()

            def wait_connect(addr):
                connected.append(addr)
                connections.get()
            mock_sock.connect = wait_connect

            memcache_client = memcached.MemcacheRing(['1.2.3.4:11211'],
                                                     connect_timeout=10)
            # sanity
            self.assertEquals(1, len(memcache_client._client_cache))
            for server, pool in memcache_client._client_cache.items():
                self.assertEquals(2, pool.max_size)

            # make 10 requests "at the same time"
            p = GreenPool()
            for i in range(10):
                p.spawn(memcache_client.set, 'key', 'value')
            for i in range(3):
                sleep(0.1)
                self.assertEquals(2, len(connected))
            # give out a connection
            connections.put(None)
            for i in range(3):
                sleep(0.1)
                self.assertEquals(2, len(connected))
            # finish up
            for i in range(8):
                connections.put(None)
            self.assertEquals(2, len(connected))
            p.waitall()
            self.assertEquals(2, len(connected))

    def test_exhaustion(self):
        waiter = Queue(0)
        def consumer():
            gotten = None
            try:
                gotten = self.pool.get()
            finally:
                waiter.put(gotten)

        eventlet.spawn(consumer)

        one, two, three, four = (
            self.pool.get(), self.pool.get(), self.pool.get(), self.pool.get())
        self.assertEquals(self.pool.free(), 0)

        # Let consumer run; nothing will be in the pool, so he will wait
        eventlet.sleep(0)

        # Wake consumer
        self.pool.put(one)

        # wait for the consumer
        self.assertEquals(waiter.get(), one)

    def __init__(self, app, conf):
        """
        Should be placed after liteauth but before proxy-query

        """
        self.app = app
        self.conf = conf
        self.logger = get_logger(conf, log_route='lite-accounting')
        # interval between accounting data dumps
        self.interval = int(conf.get('liteacc_interval', 60))
        # how long to keep in memcache, there should be: self.interval << self.timeout
        # maybe we need: self.timeout = 100 * self.interval
        self.timeout = int(conf.get('liteacc_cache_time', 30 * 60))
        # url for accounting objects
        # Example: /v1/liteacc/accounting
        self.accounting_url = conf.get('liteacc_url', '').lower().rstrip('/')
        self.queue = Queue()
        # we will get memcache object later, with first request
        self.memcache = None
        # let's spawn the accounting thread
        spawn_n(self.accounting_server)

def run_stock_parser():
    symbol_q = Queue()
    price_q = Queue()

    stock_symbols = []
    with open('symbols.txt', 'r') as symfile:
        for n, line in enumerate(symfile):
            sym = line.strip()
            if sym:
                stock_symbols.append(sym)

    ncpu = cpu_count()

    pool = [spawn(read_stock_worker, symbol_q, price_q) for _ in range(ncpu * 2)]
    output = spawn(write_output_file, price_q)

    for symbol in stock_symbols:
        symbol_q.put(symbol)
    symbol_q.put(_sentinel)
    for p in pool:
        p.wait()
    price_q.put(_sentinel)
    output.wait()