Python defer.DeferredQueue类代码示例

class ProcessPool(object):
	def __init__(self, count=10):
		self.limiter = DeferredSemaphore(count)
		self.processes = [spawnProcess() for _ in xrange(count)]
		self.workQueue = DeferredQueue()

		for process in self.processes:
			process.onconnect.addCallback(self._prepareForWork)

	@inlineCallbacks
	def _prepareForWork(self, proto):
		deferred, func, args = yield self.workQueue.get()
		proto.queueWork(deferred, func, *args)
		def requeue(result):
			self._prepareForWork(proto)
			return result
		deferred.addCallback(requeue)
		returnValue(proto)

	def queueWork(self, function, *args):
		resultDeferred = Deferred()
		innerDeferred = Deferred()
		self.workQueue.put((innerDeferred, function, args))
		def callResult(obj):
			resultDeferred.callback(obj)
			return obj
		innerDeferred.addCallback(callResult)
		return resultDeferred

	def stop(self):
		for process in self.processes:
			process.protocol.kill()
			process.transport.loseConnection()

class WeChatTestCase(VumiTestCase):

    def setUp(self):
        self.tx_helper = self.add_helper(TransportHelper(WeChatTransport))
        self.request_queue = DeferredQueue()
        self.fake_http = FakeHttpServer(self.handle_api_request)
        self.api_url = 'https://api.wechat.com/cgi-bin/'

    def handle_api_request(self, request):
        self.assertEqual(request.path[:len(self.api_url)], self.api_url)
        self.request_queue.put(request)
        return NOT_DONE_YET

    @inlineCallbacks
    def get_transport(self, **config):
        defaults = {
            'auth_token': 'token',
            'twisted_endpoint': 'tcp:0',
            'wechat_appid': 'appid',
            'wechat_secret': 'secret',
            'embed_user_profile': False,
        }
        defaults.update(config)
        transport = yield self.tx_helper.get_transport(defaults)
        transport.agent_factory = self.fake_http.get_agent
        returnValue(transport)

    @inlineCallbacks
    def get_transport_with_access_token(self, access_token, **config):
        transport = yield self.get_transport(**config)
        yield transport.redis.set(WeChatTransport.ACCESS_TOKEN_KEY,
                                  access_token)
        returnValue(transport)

class MockHttpServer(object):
    """
    NOTE: This is deprecated.
          Please use :class:`vumi.tests.http_helpers.MockHttpHelper` instead.
    """

    def __init__(self, handler=None):
        self.queue = DeferredQueue()
        self._handler = handler or self.handle_request
        self._webserver = None
        self.addr = None
        self.url = None

    def handle_request(self, request):
        self.queue.put(request)

    @inlineCallbacks
    def start(self):
        root = MockResource(self._handler)
        site_factory = LogFilterSite(root)
        self._webserver = yield reactor.listenTCP(
            0, site_factory, interface='127.0.0.1')
        self.addr = self._webserver.getHost()
        self.url = "http://%s:%s/" % (self.addr.host, self.addr.port)

    @inlineCallbacks
    def stop(self):
        yield self._webserver.stopListening()
        yield self._webserver.loseConnection()

class TestHTTPClientBase(TestCase):
    # TODO: Run client tests synchronously with treq.testing tools (#38)
    run_tests_with = AsynchronousDeferredRunTest.make_factory(timeout=0.1)

    def setUp(self):
        super(TestHTTPClientBase, self).setUp()

        self.requests = DeferredQueue()
        self.fake_server = FakeHttpServer(self.handle_request)

        fake_client = treq_HTTPClient(self.fake_server.get_agent())
        self.client = self.get_client(fake_client)

        # Spin the reactor once at the end of each test to clean up any
        # cancelled deferreds
        self.addCleanup(wait0)

    def handle_request(self, request):
        self.requests.put(request)
        return NOT_DONE_YET

    def get_client(self, client):
        """To be implemented by subclass"""
        raise NotImplementedError()

    def uri(self, path):
        return '%s%s' % (self.client.url, path,)

    def cleanup_d(self, d):
        self.addCleanup(lambda: d)
        return d

def event_queue(event):
    q = DeferredQueue()
    def cb(*args, **kwargs):
        q.put((args, kwargs))
    h = event.subscribe_repeating(cb)
    q.unsubscribe = h.unsubscribe
    return q

class QueuePoller(object):
    implements(IPoller)

    def __init__(self, settings):
        self.q = RedisSpiderQueue(settings)
        self.dq = DeferredQueue(size=1)

    @inlineCallbacks
    def poll(self):
        if self.dq.pending:
            return
        c = yield maybeDeferred(self.q.count)
        if c:
            msg = yield maybeDeferred(self.q.pop)
            returnValue(self.dq.put(self._message(msg)))

    def next(self):
        return self.dq.get()

    def _message(self, queue_msg):
        d = queue_msg.copy()
        d['_project'] = SCRAPY_PROJECT
        d['_spider'] = d.pop('name')
        d['_job'] = d.pop('jobid', uuid.uuid1().hex)
        return d

class SSMIServerProtocol(Protocol):
    delimiter = TruteqTransportProtocol.delimiter

    def __init__(self):
        self.receive_queue = DeferredQueue()
        self._buf = b""

    def dataReceived(self, data):
        self._buf += data
        self.parse_commands()

    def parse_commands(self):
        while self.delimiter in self._buf:
            line, _, self._buf = self._buf.partition(self.delimiter)
            if line:
                self.receive_queue.put(SSMIRequest.parse(line))

    def send(self, command):
        self.transport.write(str(command))
        self.transport.write(self.delimiter)
        return wait0()

    def receive(self):
        return self.receive_queue.get()

    def disconnect(self):
        self.transport.loseConnection()

class QueuePoller(object):

    def __init__(self, config):
        self.config = config
        self.update_projects()
        self.dq = DeferredQueue()

    @inlineCallbacks
    def poll(self):
        if not self.dq.waiting:
            return
        for p, q in iteritems(self.queues):
            c = yield maybeDeferred(q.count)
            if c:
                msg = yield maybeDeferred(q.pop)
                if msg is not None:  # In case of a concurrently accessed queue
                    returnValue(self.dq.put(self._message(msg, p)))

    def next(self):
        return self.dq.get()

    def update_projects(self):
        self.queues = get_spider_queues(self.config)

    def _message(self, queue_msg, project):
        d = queue_msg.copy()
        d['_project'] = project
        d['_spider'] = d.pop('name')
        return d

    def test_health_response(self):
        health_url = 'http://%s:%s%s' % (
            self.addr.host, self.addr.port, self.config['health_path'])

        response = yield http_request_full(health_url, method='GET')
        self.assertEqual(response.delivered_body, '0')

        yield self.app_helper.make_dispatch_inbound(
            'in 1', message_id='1', conv=self.conversation)

        queue = DeferredQueue()
        stream_url = '%s/%s/messages.json' % (self.url, self.conversation.key)
        stream_receiver = self.client.stream(
            TransportUserMessage, queue.put, queue.put, stream_url,
            Headers(self.auth_headers))

        yield queue.get()

        response = yield http_request_full(health_url, method='GET')
        self.assertEqual(response.delivered_body, '1')

        stream_receiver.disconnect()

        response = yield http_request_full(health_url, method='GET')
        self.assertEqual(response.delivered_body, '0')

        self.assertEqual(self.app.client_manager.clients, {
            'sphex.stream.message.%s' % (self.conversation.key,): []
        })

    def test_events_stream(self):
        url = '%s/%s/events.json' % (self.url, self.conversation.key)

        events = DeferredQueue()
        errors = DeferredQueue()
        receiver = yield self.client.stream(TransportEvent, events.put,
                                            events.put, url,
                                            Headers(self.auth_headers))

        msg1 = yield self.app_helper.make_stored_outbound(
            self.conversation, 'out 1', message_id='1')
        ack1 = yield self.app_helper.make_dispatch_ack(
            msg1, conv=self.conversation)

        msg2 = yield self.app_helper.make_stored_outbound(
            self.conversation, 'out 2', message_id='2')
        ack2 = yield self.app_helper.make_dispatch_ack(
            msg2, conv=self.conversation)

        ra1 = yield events.get()
        ra2 = yield events.get()

        receiver.disconnect()

        self.assertEqual(ack1['event_id'], ra1['event_id'])
        self.assertEqual(ack2['event_id'], ra2['event_id'])
        self.assertEqual(errors.size, None)

    def test_messages_stream(self):
        url = '%s/%s/messages.json' % (self.url, self.conversation.key)

        messages = DeferredQueue()
        errors = DeferredQueue()
        receiver = self.client.stream(
            TransportUserMessage, messages.put, errors.put, url,
            Headers(self.auth_headers))

        msg1 = yield self.app_helper.make_dispatch_inbound(
            'in 1', message_id='1', conv=self.conversation)

        msg2 = yield self.app_helper.make_dispatch_inbound(
            'in 2', message_id='2', conv=self.conversation)

        rm1 = yield messages.get()
        rm2 = yield messages.get()

        receiver.disconnect()

        # Sometimes messages arrive out of order if we're hitting real redis.
        rm1, rm2 = sorted([rm1, rm2], key=lambda m: m['message_id'])

        self.assertEqual(msg1['message_id'], rm1['message_id'])
        self.assertEqual(msg2['message_id'], rm2['message_id'])
        self.assertEqual(errors.size, None)

class WeChatTestCase(VumiTestCase):

    def setUp(self):
        self.tx_helper = self.add_helper(TransportHelper(WeChatTransport))
        self.request_queue = DeferredQueue()
        self.mock_server = MockHttpServer(self.handle_api_request)
        self.add_cleanup(self.mock_server.stop)
        return self.mock_server.start()

    def handle_api_request(self, request):
        self.request_queue.put(request)
        return NOT_DONE_YET

    def get_transport(self, **config):
        defaults = {
            'api_url': self.mock_server.url,
            'auth_token': 'token',
            'twisted_endpoint': 'tcp:0',
            'wechat_appid': 'appid',
            'wechat_secret': 'secret',
            'embed_user_profile': False,
        }
        defaults.update(config)
        return self.tx_helper.get_transport(defaults)

    @inlineCallbacks
    def get_transport_with_access_token(self, access_token, **config):
        transport = yield self.get_transport(**config)
        yield transport.redis.set(WeChatTransport.ACCESS_TOKEN_KEY,
                                  access_token)
        returnValue(transport)

class QueuePoller(object):

    implements(IPoller)

    def __init__(self, config):
        self.config = config
        self.update_projects()
        self.dq = DeferredQueue(size=1)

    @inlineCallbacks
    def poll(self):
        if self.dq.pending:
            return
        for p, q in self.queues.iteritems():
            c = yield maybeDeferred(q.count)
            if c:
                msg = yield maybeDeferred(q.pop)
                returnValue(self.dq.put(self._message(msg, p)))

    def next(self):
        return self.dq.get()

    def update_projects(self):
        self.queues = get_spider_queues(self.config)

    def _message(self, queue_msg, project):
        d = queue_msg.copy()
        d['_project'] = project
        d['_spider'] = d.pop('name')
        return d

 def async_receive_stream(self, func, *args, **kw):
     queue = DeferredQueue()
     def _execute():
         for result in func(*args, **kw):
             reactor.callFromThread(queue.put, result)
     _ = threads.deferToThread(_execute)
     while 1:
         yield queue.get()

    def __init__(self):
        """Create a new TCP proxy.

        `self.server_queue` contains messages from end server to client.
        `self.client_queue` contains messages from client to end server.
        """
        self.server_queue = DeferredQueue()
        self.client_queue = DeferredQueue()
        self.server_queue.get().addCallback(self.serverQueueCallback)