当前位置: 首页>>代码示例>>Python>>正文


Python defer.DeferredQueue类代码示例

本文整理汇总了Python中twisted.internet.defer.DeferredQueue的典型用法代码示例。如果您正苦于以下问题:Python DeferredQueue类的具体用法?Python DeferredQueue怎么用?Python DeferredQueue使用的例子?那么恭喜您, 这里精选的类代码示例或许可以为您提供帮助。


在下文中一共展示了DeferredQueue类的15个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Python代码示例。

示例1: ProcessPool

class ProcessPool(object):
	def __init__(self, count=10):
		self.limiter = DeferredSemaphore(count)
		self.processes = [spawnProcess() for _ in xrange(count)]
		self.workQueue = DeferredQueue()

		for process in self.processes:
			process.onconnect.addCallback(self._prepareForWork)

	@inlineCallbacks
	def _prepareForWork(self, proto):
		deferred, func, args = yield self.workQueue.get()
		proto.queueWork(deferred, func, *args)
		def requeue(result):
			self._prepareForWork(proto)
			return result
		deferred.addCallback(requeue)
		returnValue(proto)

	def queueWork(self, function, *args):
		resultDeferred = Deferred()
		innerDeferred = Deferred()
		self.workQueue.put((innerDeferred, function, args))
		def callResult(obj):
			resultDeferred.callback(obj)
			return obj
		innerDeferred.addCallback(callResult)
		return resultDeferred

	def stop(self):
		for process in self.processes:
			process.protocol.kill()
			process.transport.loseConnection()
开发者ID:systocrat,项目名称:armada,代码行数:33,代码来源:pool.py

示例2: WeChatTestCase

class WeChatTestCase(VumiTestCase):

    def setUp(self):
        self.tx_helper = self.add_helper(TransportHelper(WeChatTransport))
        self.request_queue = DeferredQueue()
        self.fake_http = FakeHttpServer(self.handle_api_request)
        self.api_url = 'https://api.wechat.com/cgi-bin/'

    def handle_api_request(self, request):
        self.assertEqual(request.path[:len(self.api_url)], self.api_url)
        self.request_queue.put(request)
        return NOT_DONE_YET

    @inlineCallbacks
    def get_transport(self, **config):
        defaults = {
            'auth_token': 'token',
            'twisted_endpoint': 'tcp:0',
            'wechat_appid': 'appid',
            'wechat_secret': 'secret',
            'embed_user_profile': False,
        }
        defaults.update(config)
        transport = yield self.tx_helper.get_transport(defaults)
        transport.agent_factory = self.fake_http.get_agent
        returnValue(transport)

    @inlineCallbacks
    def get_transport_with_access_token(self, access_token, **config):
        transport = yield self.get_transport(**config)
        yield transport.redis.set(WeChatTransport.ACCESS_TOKEN_KEY,
                                  access_token)
        returnValue(transport)
开发者ID:caiobertacco,项目名称:vumi,代码行数:33,代码来源:test_wechat.py

示例3: MockHttpServer

class MockHttpServer(object):
    """
    NOTE: This is deprecated.
          Please use :class:`vumi.tests.http_helpers.MockHttpHelper` instead.
    """

    def __init__(self, handler=None):
        self.queue = DeferredQueue()
        self._handler = handler or self.handle_request
        self._webserver = None
        self.addr = None
        self.url = None

    def handle_request(self, request):
        self.queue.put(request)

    @inlineCallbacks
    def start(self):
        root = MockResource(self._handler)
        site_factory = LogFilterSite(root)
        self._webserver = yield reactor.listenTCP(
            0, site_factory, interface='127.0.0.1')
        self.addr = self._webserver.getHost()
        self.url = "http://%s:%s/" % (self.addr.host, self.addr.port)

    @inlineCallbacks
    def stop(self):
        yield self._webserver.stopListening()
        yield self._webserver.loseConnection()
开发者ID:AndrewCvekl,项目名称:vumi,代码行数:29,代码来源:utils.py

示例4: TestHTTPClientBase

class TestHTTPClientBase(TestCase):
    # TODO: Run client tests synchronously with treq.testing tools (#38)
    run_tests_with = AsynchronousDeferredRunTest.make_factory(timeout=0.1)

    def setUp(self):
        super(TestHTTPClientBase, self).setUp()

        self.requests = DeferredQueue()
        self.fake_server = FakeHttpServer(self.handle_request)

        fake_client = treq_HTTPClient(self.fake_server.get_agent())
        self.client = self.get_client(fake_client)

        # Spin the reactor once at the end of each test to clean up any
        # cancelled deferreds
        self.addCleanup(wait0)

    def handle_request(self, request):
        self.requests.put(request)
        return NOT_DONE_YET

    def get_client(self, client):
        """To be implemented by subclass"""
        raise NotImplementedError()

    def uri(self, path):
        return '%s%s' % (self.client.url, path,)

    def cleanup_d(self, d):
        self.addCleanup(lambda: d)
        return d
开发者ID:praekeltfoundation,项目名称:marathon-acme,代码行数:31,代码来源:test_clients.py

示例5: event_queue

def event_queue(event):
    q = DeferredQueue()
    def cb(*args, **kwargs):
        q.put((args, kwargs))
    h = event.subscribe_repeating(cb)
    q.unsubscribe = h.unsubscribe
    return q
开发者ID:PR2,项目名称:linux_networking,代码行数:7,代码来源:async_helpers.py

示例6: QueuePoller

class QueuePoller(object):
    implements(IPoller)

    def __init__(self, settings):
        self.q = RedisSpiderQueue(settings)
        self.dq = DeferredQueue(size=1)

    @inlineCallbacks
    def poll(self):
        if self.dq.pending:
            return
        c = yield maybeDeferred(self.q.count)
        if c:
            msg = yield maybeDeferred(self.q.pop)
            returnValue(self.dq.put(self._message(msg)))

    def next(self):
        return self.dq.get()

    def _message(self, queue_msg):
        d = queue_msg.copy()
        d['_project'] = SCRAPY_PROJECT
        d['_spider'] = d.pop('name')
        d['_job'] = d.pop('jobid', uuid.uuid1().hex)
        return d
开发者ID:LightKool,项目名称:scraper,代码行数:25,代码来源:poller.py

示例7: SSMIServerProtocol

class SSMIServerProtocol(Protocol):
    delimiter = TruteqTransportProtocol.delimiter

    def __init__(self):
        self.receive_queue = DeferredQueue()
        self._buf = b""

    def dataReceived(self, data):
        self._buf += data
        self.parse_commands()

    def parse_commands(self):
        while self.delimiter in self._buf:
            line, _, self._buf = self._buf.partition(self.delimiter)
            if line:
                self.receive_queue.put(SSMIRequest.parse(line))

    def send(self, command):
        self.transport.write(str(command))
        self.transport.write(self.delimiter)
        return wait0()

    def receive(self):
        return self.receive_queue.get()

    def disconnect(self):
        self.transport.loseConnection()
开发者ID:caiobertacco,项目名称:vumi,代码行数:27,代码来源:test_truteq.py

示例8: QueuePoller

class QueuePoller(object):

    def __init__(self, config):
        self.config = config
        self.update_projects()
        self.dq = DeferredQueue()

    @inlineCallbacks
    def poll(self):
        if not self.dq.waiting:
            return
        for p, q in iteritems(self.queues):
            c = yield maybeDeferred(q.count)
            if c:
                msg = yield maybeDeferred(q.pop)
                if msg is not None:  # In case of a concurrently accessed queue
                    returnValue(self.dq.put(self._message(msg, p)))

    def next(self):
        return self.dq.get()

    def update_projects(self):
        self.queues = get_spider_queues(self.config)

    def _message(self, queue_msg, project):
        d = queue_msg.copy()
        d['_project'] = project
        d['_spider'] = d.pop('name')
        return d
开发者ID:scrapy,项目名称:scrapyd,代码行数:29,代码来源:poller.py

示例9: test_health_response

    def test_health_response(self):
        health_url = 'http://%s:%s%s' % (
            self.addr.host, self.addr.port, self.config['health_path'])

        response = yield http_request_full(health_url, method='GET')
        self.assertEqual(response.delivered_body, '0')

        yield self.app_helper.make_dispatch_inbound(
            'in 1', message_id='1', conv=self.conversation)

        queue = DeferredQueue()
        stream_url = '%s/%s/messages.json' % (self.url, self.conversation.key)
        stream_receiver = self.client.stream(
            TransportUserMessage, queue.put, queue.put, stream_url,
            Headers(self.auth_headers))

        yield queue.get()

        response = yield http_request_full(health_url, method='GET')
        self.assertEqual(response.delivered_body, '1')

        stream_receiver.disconnect()

        response = yield http_request_full(health_url, method='GET')
        self.assertEqual(response.delivered_body, '0')

        self.assertEqual(self.app.client_manager.clients, {
            'sphex.stream.message.%s' % (self.conversation.key,): []
        })
开发者ID:ChrisNolan1992,项目名称:vumi-go,代码行数:29,代码来源:test_vumi_app.py

示例10: test_events_stream

    def test_events_stream(self):
        url = '%s/%s/events.json' % (self.url, self.conversation.key)

        events = DeferredQueue()
        errors = DeferredQueue()
        receiver = yield self.client.stream(TransportEvent, events.put,
                                            events.put, url,
                                            Headers(self.auth_headers))

        msg1 = yield self.app_helper.make_stored_outbound(
            self.conversation, 'out 1', message_id='1')
        ack1 = yield self.app_helper.make_dispatch_ack(
            msg1, conv=self.conversation)

        msg2 = yield self.app_helper.make_stored_outbound(
            self.conversation, 'out 2', message_id='2')
        ack2 = yield self.app_helper.make_dispatch_ack(
            msg2, conv=self.conversation)

        ra1 = yield events.get()
        ra2 = yield events.get()

        receiver.disconnect()

        self.assertEqual(ack1['event_id'], ra1['event_id'])
        self.assertEqual(ack2['event_id'], ra2['event_id'])
        self.assertEqual(errors.size, None)
开发者ID:ChrisNolan1992,项目名称:vumi-go,代码行数:27,代码来源:test_vumi_app.py

示例11: test_messages_stream

    def test_messages_stream(self):
        url = '%s/%s/messages.json' % (self.url, self.conversation.key)

        messages = DeferredQueue()
        errors = DeferredQueue()
        receiver = self.client.stream(
            TransportUserMessage, messages.put, errors.put, url,
            Headers(self.auth_headers))

        msg1 = yield self.app_helper.make_dispatch_inbound(
            'in 1', message_id='1', conv=self.conversation)

        msg2 = yield self.app_helper.make_dispatch_inbound(
            'in 2', message_id='2', conv=self.conversation)

        rm1 = yield messages.get()
        rm2 = yield messages.get()

        receiver.disconnect()

        # Sometimes messages arrive out of order if we're hitting real redis.
        rm1, rm2 = sorted([rm1, rm2], key=lambda m: m['message_id'])

        self.assertEqual(msg1['message_id'], rm1['message_id'])
        self.assertEqual(msg2['message_id'], rm2['message_id'])
        self.assertEqual(errors.size, None)
开发者ID:ChrisNolan1992,项目名称:vumi-go,代码行数:26,代码来源:test_vumi_app.py

示例12: WeChatTestCase

class WeChatTestCase(VumiTestCase):

    def setUp(self):
        self.tx_helper = self.add_helper(TransportHelper(WeChatTransport))
        self.request_queue = DeferredQueue()
        self.mock_server = MockHttpServer(self.handle_api_request)
        self.add_cleanup(self.mock_server.stop)
        return self.mock_server.start()

    def handle_api_request(self, request):
        self.request_queue.put(request)
        return NOT_DONE_YET

    def get_transport(self, **config):
        defaults = {
            'api_url': self.mock_server.url,
            'auth_token': 'token',
            'twisted_endpoint': 'tcp:0',
            'wechat_appid': 'appid',
            'wechat_secret': 'secret',
            'embed_user_profile': False,
        }
        defaults.update(config)
        return self.tx_helper.get_transport(defaults)

    @inlineCallbacks
    def get_transport_with_access_token(self, access_token, **config):
        transport = yield self.get_transport(**config)
        yield transport.redis.set(WeChatTransport.ACCESS_TOKEN_KEY,
                                  access_token)
        returnValue(transport)
开发者ID:Nagato23,项目名称:vumi,代码行数:31,代码来源:test_wechat.py

示例13: QueuePoller

class QueuePoller(object):

    implements(IPoller)

    def __init__(self, config):
        self.config = config
        self.update_projects()
        self.dq = DeferredQueue(size=1)

    @inlineCallbacks
    def poll(self):
        if self.dq.pending:
            return
        for p, q in self.queues.iteritems():
            c = yield maybeDeferred(q.count)
            if c:
                msg = yield maybeDeferred(q.pop)
                returnValue(self.dq.put(self._message(msg, p)))

    def next(self):
        return self.dq.get()

    def update_projects(self):
        self.queues = get_spider_queues(self.config)

    def _message(self, queue_msg, project):
        d = queue_msg.copy()
        d['_project'] = project
        d['_spider'] = d.pop('name')
        return d
开发者ID:535521469,项目名称:crawler_sth,代码行数:30,代码来源:poller.py

示例14: async_receive_stream

 def async_receive_stream(self, func, *args, **kw):
     queue = DeferredQueue()
     def _execute():
         for result in func(*args, **kw):
             reactor.callFromThread(queue.put, result)
     _ = threads.deferToThread(_execute)
     while 1:
         yield queue.get()
开发者ID:gcgirish-radisys,项目名称:voltha,代码行数:8,代码来源:streaming_client.py

示例15: __init__

    def __init__(self):
        """Create a new TCP proxy.

        `self.server_queue` contains messages from end server to client.
        `self.client_queue` contains messages from client to end server.
        """
        self.server_queue = DeferredQueue()
        self.client_queue = DeferredQueue()
        self.server_queue.get().addCallback(self.serverQueueCallback)
开发者ID:naphatkrit,项目名称:TigerHost,代码行数:9,代码来源:tcp_proxy.py


注:本文中的twisted.internet.defer.DeferredQueue类示例由纯净天空整理自Github/MSDocs等开源代码及文档管理平台,相关代码片段筛选自各路编程大神贡献的开源项目,源码版权归原作者所有,传播和使用请参考对应项目的License;未经允许,请勿转载。