Python queue.Queue类代码示例

    def websocket_protocol(self, req):
        """Runs the WebSocket protocol after the handshake is complete.

        Creates two `Queue` instances for incoming and outgoing messages and
        passes them to the `web_socket_handler` that was supplied to the
        `WebSocketServer` constructor.

        """
        inq = Queue()
        outq = Queue()

        def wrap(req, inq, outq):
            self.web_socket_handler(req, inq, outq)
            outq.put(WebSocketDisconnect())

        handler_loop = fork(wrap, req, inq, outq)

        if req.rfc_handshake:
            handle_frames = self.handle_rfc_6455_frames
        else:
            handle_frames = self.handle_non_rfc_frames

        try:
            handle_frames(inq, outq)
        except ConnectionClosed:
            if handler_loop.running:
                inq.put(WebSocketDisconnect())
            raise

    def __init__(self, uri, logger=None, log_level=None):
        self.uri = uri
        self.zmq_socket = None
        self.log = logger or None
        self.selected_log_level = log_level
        self.clients = {}
        self.outgoing = Queue()
        self.incoming = Queue()
        self.name = self.name or self.__class__.__name__
        self._incoming_loop = None

        # Allow for custom `should_run` properties in subclasses.
        try:
            self.should_run = True
        except AttributeError:
            # A custom `should_run` property exists.
            pass

        if self.log and self.selected_log_level is not None:
            self.selected_log_level = None
            warnings.warn(
                "ignored `log_level` argument since `logger` was provided.",
                RuntimeWarning,
                stacklevel=2,
            )

class TestQueueTimeouts(object):
    def setup(self):
        self.result = Event()
        self.queue = Queue()
        self.timeouts = 0
        diesel.fork(self.consumer, 0.01)
        diesel.fork(self.producer, 0.05)
        diesel.fork(self.consumer, 0.10)
        ev, val = diesel.first(sleep=TIMEOUT, waits=[self.result])
        if ev == 'sleep':
            assert 0, 'timed out'

    def consumer(self, timeout):
        try:
            self.queue.get(timeout=timeout)
            self.result.set()
        except QueueTimeout:
            self.timeouts += 1

    def producer(self, delay):
        diesel.sleep(delay)
        self.queue.put('test')

    def test_a_consumer_timed_out(self):
        assert self.timeouts == 1

    def test_a_consumer_got_a_value(self):
        assert self.result.is_set

class ChatClient(Client):
    def __init__(self, *args, **kw):
        Client.__init__(self, *args, **kw)
        self.input = Queue()

    def read_chat_message(self, prompt):
        msg = raw_input(prompt)
        return msg

    def input_handler(self):
        nick = thread(self.read_chat_message, "nick: ").strip()
        self.nick = nick
        self.input.put(nick)
        while True:
            msg = thread(self.read_chat_message, "").strip()
            self.input.put(msg)

    @call
    def chat(self):
        fork(self.input_handler)
        nick = self.input.get()
        send("%s\r\n" % nick)
        while True:
            evt, data = first(until_eol=True, waits=[self.input])
            if evt == "until_eol":
                print data.strip()
            else:
                send("%s\r\n" % data)

class ThreadPool(object):
    def __init__(self, concurrency, handler, generator, finalizer=None):
        self.concurrency = concurrency
        self.handler = handler
        self.generator = generator
        self.finalizer = finalizer

    def handler_wrap(self):
        try:
            label("thread-pool-%s" % self.handler)
            while True:
                self.waiting += 1
                if self.waiting == 1:
                    self.trigger.set()
                i = self.q.get()
                self.waiting -= 1
                if i == ThreadPoolDie:
                    return
                self.handler(i)
        finally:
            self.running -=1
            if self.waiting == 0:
                self.trigger.set()
            if self.running == 0:
                self.finished.set()

    def __call__(self):
        self.q = Queue()
        self.trigger = Event()
        self.finished = Event()
        self.waiting = 0
        self.running = 0
        try:
            while True:
                for x in xrange(self.concurrency - self.running):
                    self.running += 1
                    fork(self.handler_wrap)

                if self.waiting == 0:
                    self.trigger.wait()
                    self.trigger.clear()

                try:
                    n = self.generator()
                except StopIteration:
                    break

                self.q.put(n)
                sleep()
        finally:
            for x in xrange(self.concurrency):
                self.q.put(ThreadPoolDie)
            if self.finalizer:
                self.finished.wait()
                fork(self.finalizer)

    def websocket_protocol(self, req):
        """Runs the WebSocket protocol after the handshake is complete.

        Creates two `Queue` instances for incoming and outgoing messages and
        passes them to the `web_socket_handler` that was supplied to the
        `WebSocketServer` constructor.

        """
        inq = Queue()
        outq = Queue()

        if req.rfc_handshake:
            handle_frames = self.handle_rfc_6455_frames
        else:
            # Finish the non-RFC handshake
            key1 = req.headers.get('Sec-WebSocket-Key1')
            key2 = req.headers.get('Sec-WebSocket-Key2')

            # The final key can be in two places. The first is in the
            # `Request.data` attribute if diesel is *not* being proxied
            # to by a smart proxy that parsed HTTP requests. If it is being
            # proxied to, that data will not have been sent until after our
            # initial 101 Switching Protocols response, so we will need to
            # receive it here.

            if req.data:
                key3 = req.data
            else:
                evt, key3 = first(receive=8, sleep=5)
                assert evt == "receive", "timed out while finishing handshake"

            num1 = int(''.join(c for c in key1 if c in '0123456789'))
            num2 = int(''.join(c for c in key2 if c in '0123456789'))
            assert num1 % key1.count(' ') == 0
            assert num2 % key2.count(' ') == 0
            final = pack('!II8s', num1 / key1.count(' '), num2 / key2.count(' '), key3)
            handshake_finish = hashlib.md5(final).digest()
            send(handshake_finish)

            handle_frames = self.handle_non_rfc_frames

        def wrap(req, inq, outq):
            self.web_socket_handler(req, inq, outq)
            outq.put(WebSocketDisconnect())

        handler_loop = fork(wrap, req, inq, outq)

        try:
            handle_frames(inq, outq)
        except ConnectionClosed:
            if handler_loop.running:
                inq.put(WebSocketDisconnect())
            raise

def test_pending_events_dont_break_ordering_when_handling_early_values():

    # This test confirms that "early values" returned from a Waiter do
    # not give other pending event sources the chance to switch their
    # values into the greenlet while it context switches to give other
    # greenlets a chance to run.

    # First we setup a fake connection. It mimics a connection that does
    # not have data waiting in the buffer, and has to wait for the system
    # to call it back when data is ready on the socket. The delay argument
    # specifies how long the test should wait before simulating that data
    # is ready.

    conn1 = FakeConnection(1, delay=[None, 0.1])

    # Next we setup a Queue instance and prime it with a value, so it will
    # be ready early and return an EarlyValue.

    q = Queue()
    q.put(1)

    # Force our fake connection into the connection stack for the current
    # loop so we can make network calls (like until_eol).

    loop = core.current_loop
    loop.connection_stack.append(conn1)

    try:

        # OK, this first() call does two things.
        # 1) It calls until_eol, finds that no data is ready, and sets up a
        #    callback to be triggered when data is ready (which our
        #    FakeConnection will simulate).
        # 2) Fetches from the 'q' which will result in an EarlyValue.

        source, value = diesel.first(until_eol=True, waits=[q])
        assert source == q, source

        # What must happen is that the callback registered to handle data
        # from the FakeConnection when it arrives MUST BE CANCELED/DISCARDED/
        # FORGOTTEN/NEVER CALLED. If it gets called, it will muck with
        # internal state, and possibly switch back into the running greenlet
        # with an unexpected value, which will throw off the ordering of
        # internal state and basically break everything.

        v = diesel.until_eol()
        assert v == 'expected value 1\r\n', 'actual value == %r !!!' % (v,)

    finally:
        loop.connection_stack = []

    def do_upgrade(self, req):
        if req.headers.get_one('Upgrade') != 'WebSocket':
            return self.web_handler(req)

        # do upgrade response
        org = req.headers.get_one('Origin')

        send(
'''HTTP/1.1 101 Web Socket Protocol Handshake\r
Upgrade: WebSocket\r
Connection: Upgrade\r
WebSocket-Origin: %s\r
WebSocket-Location: %s\r
WebSocket-Protocol: diesel-generic\r
\r
''' % (org, self.ws_location))
        
        inq = Queue()
        outq = Queue()

        def wrap(inq, outq):
            self.web_socket_handler(inq, outq)
            outq.put(WebSocketDisconnect())

        fork(wrap, inq, outq)
                                    
        while True:
            try:
                typ, val = first(receive=1, waits=[outq.wait_id])
                if typ == 'receive':
                    assert val == '\x00'
                    val = until('\xff')[:-1]
                    if val == '':
                        inq.put(WebSocketDisconnect())
                    else:
                        data = dict((k, v[0]) if len(v) == 1 else (k, v) for k, v in cgi.parse_qs(val).iteritems())
                        inq.put(WebSocketData(data))
                else:
                    try:
                        v = outq.get(waiting=False)
                    except QueueEmpty:
                        pass
                    else:
                        if type(v) is WebSocketDisconnect:
                            send('\x00\xff')
                            break
                        else:
                            data = dumps(dict(v))
                            send('\x00%s\xff' % data)

            except ConnectionClosed:
                inq.put(WebSocketDisconnect())
                raise ConnectionClosed("remote disconnected")

 def __init__(self, uri, logger=None, log_level=None):
     self.uri = uri
     self.zmq_socket = None
     self.log = logger or None
     self.selected_log_level = log_level
     self.clients = {}
     self.outgoing = Queue()
     self.incoming = Queue()
     self.name = self.name or self.__class__.__name__
     if self.log and self.selected_log_level is not None:
         self.selected_log_level = None
         warnings.warn(
             "ignored `log_level` argument since `logger` was provided.",
             RuntimeWarning,
             stacklevel=2,
         )

    def __call__(self):
        self.q = Queue()
        self.trigger = Event()
        self.finished = Event()
        self.waiting = 0
        self.running = 0
        try:
            while True:
                for x in xrange(self.concurrency - self.running):
                    self.running += 1
                    fork(self.handler_wrap)

                if self.waiting == 0:
                    self.trigger.wait()
                    self.trigger.clear()

                try:
                    n = self.generator()
                except StopIteration:
                    break

                self.q.put(n)
                sleep()
        finally:
            for x in xrange(self.concurrency):
                self.q.put(ThreadPoolDie)
            if self.finalizer:
                self.finished.wait()
                fork(self.finalizer)

class ProcessPool(object):
    """A bounded pool of subprocesses.

    An instance is callable, just like a Process, and will return the result
    of executing the function in a subprocess. If all subprocesses are busy,
    the caller will wait in a queue.

    """
    def __init__(self, concurrency, handler):
        """Creates a new ProcessPool with subprocesses that run the handler.

        Args:
            concurrency (int): The number of subprocesses to spawn.
            handler (callable): A callable that the subprocesses will execute.

        """
        self.concurrency = concurrency
        self.handler = handler
        self.available_procs = Queue()
        self.all_procs = []

    def __call__(self, *args, **params):
        """Gets a process from the pool, executes it, and returns the results.

        This call will block until there is a process available to handle it.

        """
        if not self.all_procs:
            raise NoSubProcesses("Did you forget to start the pool?")
        try:
            p = self.available_procs.get()
            result = p(*args, **params)
            return result
        finally:
            self.available_procs.put(p)

    def pool(self):
        """A callable that starts the processes in the pool.

        This is useful as the callable to pass to a diesel.Loop when adding a
        ProcessPool to your application.

        """
        for i in xrange(self.concurrency):
            proc = spawn(self.handler)
            self.available_procs.put(proc)
            self.all_procs.append(proc)

 def setup(self):
     self.queue = Queue()
     self.done = Countdown(N)
     self.results = []
     self.handled = defaultdict(int)
     self.populate()
     self.consume()
     self.trigger()

    def __init__(self, concurrency, handler):
        """Creates a new ProcessPool with subprocesses that run the handler.

        Args:
            concurrency (int): The number of subprocesses to spawn.
            handler (callable): A callable that the subprocesses will execute.

        """
        self.concurrency = concurrency
        self.handler = handler
        self.available_procs = Queue()
        self.all_procs = []

    def network_set(self, client, key, value, new):
        proposal_id = idgen.next()
        resq = Queue()
        if new:
            rollback = '|' + new + ':' + proposal_id
            value += rollback
        else:
            rollback = None

        for q in self.proposal_qs:
            q.put((proposal_id, key, resq))

        success = 0
        while True: # XXX timeout etc
            v = resq.get()
            if v == PROPOSE_SUCCESS:
                success += 1
                if success == self.quorum_size:
                    break
            elif v == PROPOSE_FAIL:
                return None
            else:
                assert 0

        for q in self.save_qs:
            q.put((proposal_id, key, value, client, rollback, resq))

        success = 0
        while True: # XXX timeout etc
            v = resq.get()
            if v == PROPOSE_SUCCESS:
                pass # don't care
            elif v == PROPOSE_FAIL:
                pass # don't care
            elif v == SAVE_SUCCESS:
                success += 1
                if success == self.quorum_size:
                    return proposal_id
            else:
                assert 0

    def get_many(self, keys, concurrency_limit=100, no_failures=False):
        assert self.used_client_context, "Cannot fetch in parallel without a pooled make_client_context!"
        inq = Queue()
        outq = Queue()
        for k in keys:
            inq.put(k)

        for x in xrange(min(len(keys), concurrency_limit)):
            diesel.fork(self._subrequest, inq, outq)

        failure = False
        okay, err = [], []
        for k in keys:
            (key, success, val) = outq.get()
            if success:
                okay.append((key, val))
            else:
                err.append((key, val))

        if no_failures:
            raise BucketSubrequestException("Error in parallel subrequests", err)
        return okay, err