Python ZMQStream.stop_on_recv方法代码示例

# 需要导入模块: from zmq.eventloop.zmqstream import ZMQStream [as 别名]
# 或者: from zmq.eventloop.zmqstream.ZMQStream import stop_on_recv [as 别名]
class ManagerControlled(object):

    def __init__(self, *args, **kwargs):
        self.context = Context.instance()
        self.loop = IOLoop.instance()
        self.control_socket = self.context.socket(SUB)
        self.control_socket.setsockopt(LINGER, 0)  # discard unsent messages on close
        self.control_socket.setsockopt(SUBSCRIBE, '')
        self.control_socket.connect('tcp://{}:{}'.format(MANAGER_PUB_ADDRESS, MANAGER_PUB_PORT))
        self.control_stream = ZMQStream(self.control_socket, self.loop)
        self.control_stream.on_recv_stream(self.control_handler)

    def control_handler(self, stream, message_list):
        for message in message_list:
            try:
                notification, data = message.split()
            except ValueError:
                notification = message

            if notification == NOTIFICATION_PROCESS_STOP:
                self.stop()

    def stop(self):
        self.control_stream.stop_on_recv()
        self.control_stream.close()
        self.control_socket.close()

# 需要导入模块: from zmq.eventloop.zmqstream import ZMQStream [as 别名]
# 或者: from zmq.eventloop.zmqstream.ZMQStream import stop_on_recv [as 别名]
class MonitorEvents(SockJSConnection):
    def _zmq_msg(self, msg):
        #logging.debug(msg)
        try:
            msg_obj = json.loads(msg[0])
            logging.debug(msg_obj)
            if self.monitor != 'All':
                if 'mon_id' in msg_obj and msg_obj['mon_id'] == self.monitor:
                    self.send(msg_obj)
            else:
                self.send(msg_obj)
        except Exception as ex:
            logging.error(ex)
        
    def on_open(self, info):
        logging.debug("Monitor ticker open: "+self.monitor)
        zmq_socket = zmq.Context.instance().socket(zmq.SUB)
        zmq_socket.connect(zmq_local_endpoint)
        zmq_socket.setsockopt(zmq.SUBSCRIBE, '')
        
        self.stream = ZMQStream(zmq_socket)
        self.stream.on_recv(self._zmq_msg)

    def on_close(self):
        logging.debug("Monitor ticker close: "+self.monitor)
        self.stream.stop_on_recv()

# 需要导入模块: from zmq.eventloop.zmqstream import ZMQStream [as 别名]
# 或者: from zmq.eventloop.zmqstream.ZMQStream import stop_on_recv [as 别名]
class AsyncCircusClient(object):

    def __init__(self, context=None, endpoint=DEFAULT_ENDPOINT_DEALER,
                 timeout=5.0, ssh_server=None, ssh_keyfile=None):
        self._init_context(context)
        self.endpoint = endpoint
        self._id = b(uuid.uuid4().hex)
        self.socket = self.context.socket(zmq.DEALER)
        self.socket.setsockopt(zmq.IDENTITY, self._id)
        self.socket.setsockopt(zmq.LINGER, 0)
        get_connection(self.socket, endpoint, ssh_server, ssh_keyfile)
        self._timeout = timeout
        self.timeout = timeout * 1000
        self.stream = ZMQStream(self.socket, tornado.ioloop.IOLoop.instance())

    def _init_context(self, context):
        self.context = context or zmq.Context.instance()

    def stop(self):
        self.stream.stop_on_recv()
        # only supported by libzmq >= 3
        if hasattr(self.socket, 'disconnect'):
            self.socket.disconnect(self.endpoint)
        self.stream.close()

    @tornado.gen.coroutine
    def send_message(self, command, **props):
        res = yield self.call(make_message(command, **props))
        raise tornado.gen.Return(res)

    @tornado.gen.coroutine
    def call(self, cmd):
        if isinstance(cmd, string_types):
            raise DeprecationWarning('call() takes a mapping')

        call_id = uuid.uuid4().hex
        cmd['id'] = call_id
        try:
            cmd = json.dumps(cmd)
        except ValueError as e:
            raise CallError(str(e))

        try:
            yield tornado.gen.Task(self.stream.send, cmd)
        except zmq.ZMQError as e:
            raise CallError(str(e))

        while True:
            messages = yield tornado.gen.Task(self.stream.on_recv)
            for message in messages:
                try:
                    res = json.loads(message)
                    if res.get('id') != call_id:
                        # we got the wrong message
                        continue
                    raise tornado.gen.Return(res)
                except ValueError as e:
                    raise CallError(str(e))

# 需要导入模块: from zmq.eventloop.zmqstream import ZMQStream [as 别名]
# 或者: from zmq.eventloop.zmqstream.ZMQStream import stop_on_recv [as 别名]
class _Messenger(object):

    def __init__(self, in_sock, out_sock, context, io_loop=None):
        self._context = context
        self._io_loop = io_loop or IOLoop.instance()

        self._create_socket(in_sock, out_sock)
        self._in_stream = ZMQStream(self._in_socket, io_loop)
        self._out_stream = ZMQStream(self._out_socket, io_loop)

        self._callbacks = defaultdict(list)

    def _create_socket(self, in_sock, out_sock):
        raise NotImplementedError()

    def start(self):
        self._in_stream.on_recv(self._on_receive)

    def stop(self):
        self._in_stream.stop_on_recv()
#        self._publish(CTRL_MSG_WORKER, None, CTRL_MSG_WORKER_QUIT_ACK)
#
    def close(self):
        self._in_stream.close()
        self._in_socket.close()
        self._out_stream.close()
        self._out_socket.close()

    def _on_receive(self, zmq_msg):
        msg = CtrlMessage.deserialize(zmq_msg)

        if msg.topic in self._callbacks:
            for callback in self._callbacks[msg.topic]:
                callback(msg)

#        if msg.data == CTRL_MSG_WORKER_QUIT:
#            self.stop()

    def add_callback(self, topic, callback):
        self._callbacks[topic].append(callback)

    def remove_callback(self, topic, callback):
        if topic in self._callbacks and callback in self._callbacks[topic]:
            self._callbacks[topic].remove(callback)

    def publish(self, topic, identity, data):
        msg = CtrlMessage(topic, identity, data)
        self._out_stream.send_multipart(msg.serialize())

# 需要导入模块: from zmq.eventloop.zmqstream import ZMQStream [as 别名]
# 或者: from zmq.eventloop.zmqstream.ZMQStream import stop_on_recv [as 别名]
class AsynchronousStatsConsumer(object):
    def __init__(self, topics, loop, callback, context=None,
                 endpoint=DEFAULT_ENDPOINT_SUB, ssh_server=None, timeout=1.):
        self.topics = topics
        self.keep_context = context is not None
        self.context = context or zmq.Context()
        self.endpoint = endpoint
        self.pubsub_socket = self.context.socket(zmq.SUB)
        get_connection(self.pubsub_socket, self.endpoint, ssh_server)
        for topic in self.topics:
            self.pubsub_socket.setsockopt_string(zmq.SUBSCRIBE, topic)
        self.stream = ZMQStream(self.pubsub_socket, loop)
        self.stream.on_recv(self.process_message)
        self.callback = callback
        self.timeout = timeout

        # Connection counter
        self.count = 0

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_value, traceback):
        """ On context manager exit, destroy the zmq context """
        self.stop()

    def process_message(self, msg):
        topic, stat = msg
        stat = stat.decode('utf-8')
        topic = topic.decode('utf-8').split('.')
        if len(topic) == 3:
            __, watcher, subtopic = topic
            self.callback(watcher, subtopic, json.loads(stat), self.endpoint)
        elif len(topic) == 2:
            __, watcher = topic
            self.callback(watcher, None, json.loads(stat), self.endpoint)

    def stop(self):
        self.stream.stop_on_recv()
        if self.keep_context:
            return
        try:
            self.context.destroy(0)
        except zmq.ZMQError as e:
            if e.errno == errno.EINTR:
                pass
            else:
                raise

# 需要导入模块: from zmq.eventloop.zmqstream import ZMQStream [as 别名]
# 或者: from zmq.eventloop.zmqstream.ZMQStream import stop_on_recv [as 别名]
class LRUQueue(object):
    """LRUQueue class using ZMQStream/IOLoop for event dispatching"""
    def __init__(self, backend_socket, frontend_socket):
        self.available_workers = 0
        self.workers = []
        self.client_nbr = NBR_CLIENTS

        self.backend = ZMQStream(backend_socket)
        self.frontend = ZMQStream(frontend_socket)
        self.backend.on_recv(self.handle_backend)

        self.loop = IOLoop.instance()

    def handle_backend(self, msg):
        # Queue worker address for LRU routing
        worker_addr, empty, client_addr = msg[:3]

        assert self.available_workers < NBR_WORKERS

        self.available_workers += 1
        self.workers.append(worker_addr)

        assert empty == b""

        if client_addr != b"READY":
            empty, reply = msg[3:]

            assert empty == b""
            self.frontend.send_multipart([client_addr, b'', reply])
            self.client_nbr -= 1
            if self.client_nbr == 0:
                self.loop.add_timeout(time.time() + 1, self.loop.stop)

        if self.available_workers == 1:
            self.frontend.on_recv(self.handle_frontend)

    def handle_frontend(self, msg):
        client_addr, empty, request = msg
        assert empty == b""

        self.available_workers -= 1
        worker_id = self.workers.pop()

        self.backend.send_multipart([worker_id, b'', client_addr, b'', request])

        if self.available_workers == 0:
            self.frontend.stop_on_recv()

# 需要导入模块: from zmq.eventloop.zmqstream import ZMQStream [as 别名]
# 或者: from zmq.eventloop.zmqstream.ZMQStream import stop_on_recv [as 别名]
class LRUQueue(object):
    """LRUQueue class using ZMQStream/IOLoop for event dispatching"""

    def __init__(self, backend_socket, frontend_socket):
        self.available_workers = 0
        self.workers = []
        self.client_nbr = NBR_CLIENTS

        self.backend = ZMQStream(backend_socket)
        self.frontend = ZMQStream(frontend_socket)
        self.backend.on_recv(self.handle_backend)

        self.loop = IOLoop.instance()

    def handle_backend(self, msg):
        # Queue worker address for LRU routing
        worker_addr, empty, client_addr = msg[:3]

        assert self.available_workers < NBR_WORKERS

        # add worker back to the list of workers
        self.available_workers += 1
        self.workers.append(worker_addr)

        #   Second frame is empty
        assert empty == b""

        # Third frame is READY or else a client reply address
        # If client reply, send rest back to frontend
        if client_addr != b"READY":
            empty, reply = msg[3:]

            # Following frame is empty
            assert empty == b""

            self.frontend.send_multipart([client_addr, b'', reply])

            self.client_nbr -= 1

            if self.client_nbr == 0:
                # Exit after N messages
                self.loop.add_timeout(time.time()+1, self.loop.stop)

        if self.available_workers == 1:
            # on first recv, start accepting frontend messages
            self.frontend.on_recv(self.handle_frontend)

    def handle_frontend(self, msg):
        # Now get next client request, route to LRU worker
        # Client request is [address][empty][request]
        client_addr, empty, request = msg

        assert empty == b""

        #  Dequeue and drop the next worker address
        self.available_workers -= 1
        worker_id = self.workers.pop()

        self.backend.send_multipart([worker_id, b'', client_addr, b'', request])
        if self.available_workers == 0:
            # stop receiving until workers become available again
            self.frontend.stop_on_recv()

# 需要导入模块: from zmq.eventloop.zmqstream import ZMQStream [as 别名]
# 或者: from zmq.eventloop.zmqstream.ZMQStream import stop_on_recv [as 别名]

#.........这里部分代码省略.........
            for key,kvmsg in self.kvmap.items():
                self.flush_single(kvmsg)
    
    def flush_single(self, kvmsg):
        """If key-value pair has expired, delete it and publish the fact
        to listening clients."""
        if kvmsg.get('ttl', 0) <= time.time():
            kvmsg.body = ""
            self.sequence += 1
            kvmsg.sequence = self.sequence
            kvmsg.send(self.publisher)
            del self.kvmap[kvmsg.key]
            logging.info("I: publishing delete=%d", self.sequence)
    
    def send_hugz(self):
        """Send hugz to anyone listening on the publisher socket"""
        kvmsg = KVMsg(self.sequence)
        kvmsg.key = "HUGZ"
        kvmsg.body = ""
        kvmsg.send(self.publisher)

    # ---------------------------------------------------------------------
    # State change handlers

    def become_master(self):
        """We're becoming master

        The backup server applies its pending list to its own hash table,
        and then starts to process state snapshot requests.
        """
        self.master = True
        self.slave = False
        # stop receiving subscriber updates while we are master
        self.subscriber.stop_on_recv()
        
        # Apply pending list to own kvmap
        while self.pending:
            kvmsg = self.pending.pop(0)
            self.sequence += 1
            kvmsg.sequence = self.sequence
            kvmsg.store(self.kvmap)
            logging.info ("I: publishing pending=%d", self.sequence)


    def become_slave(self):
        """We're becoming slave"""
        # clear kvmap
        self.kvmap = None
        self.master = False
        self.slave = True
        self.subscriber.on_recv(self.handle_subscriber)
    
    def handle_subscriber(self, msg):
        """Collect updates from peer (master)
        We're always slave when we get these updates
        """
        if self.master:
            logging.warn("received subscriber message, but we are master %s", msg)
            return
        
        # Get state snapshot if necessary
        if self.kvmap is None:
            self.kvmap = {}
            snapshot = self.ctx.socket(zmq.DEALER)
            snapshot.linger = 0
            snapshot.connect("tcp://localhost:%i" % self.peer)

# 需要导入模块: from zmq.eventloop.zmqstream import ZMQStream [as 别名]
# 或者: from zmq.eventloop.zmqstream.ZMQStream import stop_on_recv [as 别名]
class ZmqWorker(object, LoggingMixin):
    """
    This is the ZMQ worker implementation.

    The worker will register a :class:`ZMQStream` with the configured
    :class:`zmq.Socket` and :class:`zmq.eventloop.ioloop.IOLoop` instance.

    Upon `ZMQStream.on_recv` the configured `processors` will be executed
    with the deserialized context and the result will be published through the
    configured `zmq.socket`.
    """

    def __init__(self, insocket, outsocket, mgmt, processing, log_handler,
            log_level, io_loop=None):
        """
        Initialize the `ZMQStream` with the `insocket` and `io_loop` and store
        the `outsocket`.

        `insocket` should be of the type `zmq.socket.PULL` `outsocket` should
        be of the type `zmq.socket.PUB`

        `mgmt` is an instance of `spyder.core.mgmt.ZmqMgmt` that handles
        communication between master and worker processes.
        """
        LoggingMixin.__init__(self, log_handler, log_level)

        self._insocket = insocket
        self._io_loop = io_loop or IOLoop.instance()
        self._outsocket = outsocket

        self._processing = processing
        self._mgmt = mgmt
        self._in_stream = ZMQStream(self._insocket, self._io_loop)
        self._out_stream = ZMQStream(self._outsocket, self._io_loop)

    def _quit(self, msg):
        """
        The worker is quitting, stop receiving messages.
        """
        if ZMQ_SPYDER_MGMT_WORKER_QUIT == msg.data:
            self.stop()

    def _receive(self, msg):
        """
        We have a message!

        `msg` is a serialized version of a `DataMessage`.
        """
        message = DataMessage(msg)

        try:
            # this is the real work we want to do
            curi = self._processing(message.curi)
            message.curi = curi
        except:
            # catch any uncaught exception and only log it as CRITICAL
            self._logger.critical(
                    "worker::Uncaught exception executing the worker for URL %s!" %
                    (message.curi.url,))
            self._logger.critical("worker::%s" % (traceback.format_exc(),))

        # finished, now send the result back to the master
        self._out_stream.send_multipart(message.serialize())

    def start(self):
        """
        Start the worker.
        """
        self._mgmt.add_callback(ZMQ_SPYDER_MGMT_WORKER, self._quit)
        self._in_stream.on_recv(self._receive)

    def stop(self):
        """
        Stop the worker.
        """
        # stop receiving
        self._in_stream.stop_on_recv()
        self._mgmt.remove_callback(ZMQ_SPYDER_MGMT_WORKER, self._quit)
        # but work on anything we might already have
        self._in_stream.flush()
        self._out_stream.flush()

    def close(self):
        """
        Close all open sockets.
        """
        self._in_stream.close()
        self._insocket.close()
        self._out_stream.close()
        self._outsocket.close()

# 需要导入模块: from zmq.eventloop.zmqstream import ZMQStream [as 别名]
# 或者: from zmq.eventloop.zmqstream.ZMQStream import stop_on_recv [as 别名]
class TornadoRPCService(RPCServiceBase):  #{
    """ An asynchronous RPC service that takes requests over a ROUTER socket.
        Using Tornado compatible IOLoop and ZMQStream from PyZMQ.
    """

    def __init__(self, context=None, ioloop=None, **kwargs):  #{
        """
        Parameters
        ==========
        ioloop : IOLoop
            An existing IOLoop instance, if not passed, zmq.IOLoop.instance()
            will be used.
        context : Context
            An existing Context instance, if not passed, zmq.Context.instance()
            will be used.
        serializer : Serializer
            An instance of a Serializer subclass that will be used to serialize
            and deserialize args, kwargs and the result.
        """
        assert context is None or isinstance(context, zmq.Context)
        self.context = context if context is not None else zmq.Context.instance()
        self.ioloop  = IOLoop.instance() if ioloop is None else ioloop
        self._is_started = False
        super(TornadoRPCService, self).__init__(**kwargs)
    #}
    def _create_socket(self):  #{
        super(TornadoRPCService, self)._create_socket()
        socket = self.context.socket(zmq.ROUTER)
        self.socket = ZMQStream(socket, self.ioloop)
    #}
    def _handle_request(self, msg_list):  #{
        """
        Handle an incoming request.

        The request is received as a multipart message:

        [<id>..<id>, b'|', req_id, proc_name, <ser_args>, <ser_kwargs>, <ignore>]

        First, the service sends back a notification that the message was
        indeed received:

        [<id>..<id>, b'|', req_id, b'ACK',  service_id]

        Next, the actual reply depends on if the call was successful or not:

        [<id>..<id>, b'|', req_id, b'OK',   <serialized result>]
        [<id>..<id>, b'|', req_id, b'FAIL', <JSON dict of ename, evalue, traceback>]

        Here the (ename, evalue, traceback) are utf-8 encoded unicode.
        """
        req = self._parse_request(msg_list)
        if req is None:
            return
        self._send_ack(req)

        ignore = req['ignore']

        try:
            # raise any parsing errors here
            if req['error']:
                raise req['error']
            # call procedure
            res = req['proc'](*req['args'], **req['kwargs'])
        except Exception:
            not ignore and self._send_fail(req)
        else:
            def send_future_result(fut):
                try:    res = fut.result()
                except: not ignore and self._send_fail(req)
                else:   not ignore and self._send_ok(req, res)

            if isinstance(res, Future):
                self.ioloop.add_future(res, send_future_result)
            else:
                not ignore and self._send_ok(req, res)
    #}
    def start(self):  #{
        """ Start the RPC service (non-blocking) """
        assert self._is_started == False, "already started"
        # register IOLoop callback
        self._is_started = True
        self.socket.on_recv(self._handle_request)
    #}
    def stop(self):  #{
        """ Stop the RPC service (non-blocking) """
        # register IOLoop callback
        self.socket.stop_on_recv()
        self._is_started = False
    #}
    def serve(self):  #{
        """ Serve RPC requests (blocking) """
        if not self._is_started:
            self.start()
        return self.ioloop.start()

# 需要导入模块: from zmq.eventloop.zmqstream import ZMQStream [as 别名]
# 或者: from zmq.eventloop.zmqstream.ZMQStream import stop_on_recv [as 别名]
class Channel(object):

    """Mother of all channels. Defines the interface.

    Callbacks:
      The callbacks will receive the channel as first parameter and
      the message as second parameter. The error callback will get
      the stream where the error occured as second parameter.

    Attributes:
      * stream_in, stream_out : the streams for eventlopp handling
      * serializer : the serializer used
    """

    def __init__(self, socket_in, socket_out, serializer):
        self.stream_in = ZMQStream(socket_in)
        self.stream_out = ZMQStream(socket_out)
        self.serializer = serializer
        self._cb_receive = None
        self._cb_send = None
        self._cb_error = None
        self._chan_id = id(self)
        return

    def on_receive(self, callback):
        """Set callback to invoke when a message was received.
        """
        self.stream_in.stop_on_recv()
        self._cb_receive = callback
        if callback:
            self.stream_in.on_recv(self._on_recv)
        return

    def on_send(self, callback):
        """Set callback to invoke when a message was sent.
        """
        self.stream_out.stop_on_send()
        self._cb_send = callback
        if callback:
            self.stream_out.on_send(self._on_send)
        return

    def on_error(self, callback):
        """Set callback to invoke when an error event occured.
        """
        self.stream_in.stop_on_err()
        self.stream_out.stop_on_err()
        self._cb_error = callback
        if callback:
            self.stream_in.on_err(self._on_err_in)
            self.stream_out.on_err(self._on_err_out)
        return

    def send(self, message):
        """Send given message.
        """
        m = self.serializer.serialize(message)
        if self.serializer.multipart:
            self.stream_out.send_multipart(m)
        else:
            self.stream_out.send(m)
        return

    def _on_recv(self, msg):
        """Helper interfacing w/ streams.
        """
        if self.serializer.multipart:
            msg = self.serializer.deserialize(msg)
        else:
            msg = self.serializer.deserialize(msg[0])
        self._cb_receive(self, msg)
        return

    def _on_send(self, sent, _):
        """Helper interfacing w/ streams.
        """
        msg = sent[0]
        if self.serializer.multipart:
             msg = self.serializer.deserialize(msg)
        else:
            msg = self.serializer.deserialize(msg[0])
        self._cb_send(self, msg)
        return

    def _on_err_in(self):
        self._cb_error(self, self.stream_in)
        return

    def _on_err_out(self):
        self._cb_error(self, self.stream_out)
        return

    def __hash__(self):
        return self._chan_id

    def __eq__(self, other):
        if not isinstance(other, self.__class__):
            return False
        return self._chan_id == other._chan_id

#.........这里部分代码省略.........

# 需要导入模块: from zmq.eventloop.zmqstream import ZMQStream [as 别名]
# 或者: from zmq.eventloop.zmqstream.ZMQStream import stop_on_recv [as 别名]
class ZmqMgmt(object):
    """
    A :class:`ZMQStream` object handling the management sockets.
    """

    def __init__(self, subscriber, publisher, **kwargs):
        """
        Initialize the management interface.

        The `subscriber` socket is the socket used by the Master to send
        commands to the workers. The publisher socket is used to send commands
        to the Master.

        You have to set the `zmq.SUBSCRIBE` socket option yourself!
        """
        self._io_loop = kwargs.get('io_loop', IOLoop.instance())

        self._subscriber = subscriber
        self._in_stream = ZMQStream(self._subscriber, self._io_loop)

        self._publisher = publisher
        self._out_stream = ZMQStream(self._publisher, self._io_loop)

        self._callbacks = dict()

    def _receive(self, raw_msg):
        """
        Main method for receiving management messages.

        `message` is a multipart message where `message[0]` contains the topic,
        `message[1]` is 0 and `message[1]` contains the actual message.
        """
        msg = MgmtMessage(raw_msg)

        if msg.topic in self._callbacks:
            for callback in self._callbacks[msg.topic]:
                if callable(callback):
                    callback(msg)

        if ZMQ_SPYDER_MGMT_WORKER_QUIT == msg.data:
            self.stop()

    def start(self):
        """
        Start the MGMT interface.
        """
        self._in_stream.on_recv(self._receive)

    def stop(self):
        """
        Stop the MGMT interface.
        """
        self._in_stream.stop_on_recv()
        self.publish(topic=ZMQ_SPYDER_MGMT_WORKER, identity=None,
                data=ZMQ_SPYDER_MGMT_WORKER_QUIT_ACK)

    def close(self):
        """
        Close all open sockets.
        """
        self._in_stream.close()
        self._subscriber.close()
        self._out_stream.close()
        self._publisher.close()

    def add_callback(self, topic, callback):
        """
        Add a callback to the specified topic.
        """
        if not callable(callback):
            raise ValueError('callback must be callable')

        if topic not in self._callbacks:
            self._callbacks[topic] = []

        self._callbacks[topic].append(callback)

    def remove_callback(self, topic, callback):
        """
        Remove a callback from the specified topic.
        """
        if topic in self._callbacks and callback in self._callbacks[topic]:
            self._callbacks[topic].remove(callback)

    def publish(self, topic=None, identity=None, data=None):
        """
        Publish a message to the intended audience.
        """
        assert topic is not None
        assert data is not None
        msg = MgmtMessage(topic=topic, identity=identity, data=data)
        self._out_stream.send_multipart(msg.serialize())

# 需要导入模块: from zmq.eventloop.zmqstream import ZMQStream [as 别名]
# 或者: from zmq.eventloop.zmqstream.ZMQStream import stop_on_recv [as 别名]

#.........这里部分代码省略.........
            else:
                zmq_pub_port = options.zmq_pub_port

            self.zmq_pub_port = zmq_pub_port

            publish_socket.bind(
                "tcp://%s:%s" % (options.zmq_pub_listen, str(self.zmq_pub_port))
            )

        # wrap pub socket into ZeroMQ stream
        self.pub_stream = ZMQStream(publish_socket)

        # create SUB socket listening to all events from all app instances
        subscribe_socket = self.zmq_context.socket(zmq.SUB)

        if self.zmq_pub_sub_proxy:
            # application started with XPUB/XSUB proxy
            self.zmq_xpub = options.zmq_xpub
            subscribe_socket.connect(self.zmq_xpub)
        else:
            # application started without XPUB/XSUB proxy
            self.zmq_sub_address = options.zmq_sub_address
            for address in self.zmq_sub_address:
                subscribe_socket.connect(address)

        subscribe_socket.setsockopt_string(
            zmq.SUBSCRIBE,
            six.u(CONTROL_CHANNEL)
        )

        subscribe_socket.setsockopt_string(
            zmq.SUBSCRIBE, six.u(ADMIN_CHANNEL)
        )

        def listen_socket():
            # wrap sub socket into ZeroMQ stream and set its on_recv callback
            self.sub_stream = ZMQStream(subscribe_socket)
            self.sub_stream.on_recv(self.dispatch_published_message)

        tornado.ioloop.IOLoop.instance().add_callback(
            listen_socket
        )

        if self.zmq_pub_sub_proxy:
            logger.info(
                "ZeroMQ XPUB: {0}, XSUB: {1}".format(self.zmq_xpub, self.zmq_xsub)
            )
        else:
            logger.info(
                "ZeroMQ PUB - {0}; subscribed to {1}".format(self.zmq_pub_port, self.zmq_sub_address)
            )

    def publish(self, channel, message, method=None):
        """
        Publish message into channel of stream.
        """
        method = method or self.DEFAULT_PUBLISH_METHOD
        message["message_type"] = method
        message = json_encode(message)
        to_publish = [utf8(channel), utf8(message)]
        self.pub_stream.send_multipart(to_publish)

    @coroutine
    def dispatch_published_message(self, multipart_message):
        """
        Got message, decide what is it and dispatch into right
        application handler.
        """
        channel = multipart_message[0]
        if six.PY3:
            channel = channel.decode()

        message_data = json_decode(multipart_message[1])

        if channel == CONTROL_CHANNEL:
            yield self.handle_control_message(message_data)
        elif channel == ADMIN_CHANNEL:
            yield self.handle_admin_message(message_data)
        else:
            yield self.handle_channel_message(channel, message_data)

    def subscribe_key(self, subscription_key):
        self.sub_stream.setsockopt_string(
            zmq.SUBSCRIBE, six.u(subscription_key)
        )

    def unsubscribe_key(self, subscription_key):
        self.sub_stream.setsockopt_string(
            zmq.UNSUBSCRIBE, six.u(subscription_key)
        )

    def clean(self):
        """
        Properly close ZeroMQ sockets.
        """
        if hasattr(self, 'pub_stream') and self.pub_stream:
            self.pub_stream.close()
        if hasattr(self, 'sub_stream') and self.sub_stream:
            self.sub_stream.stop_on_recv()
            self.sub_stream.close()

# 需要导入模块: from zmq.eventloop.zmqstream import ZMQStream [as 别名]
# 或者: from zmq.eventloop.zmqstream.ZMQStream import stop_on_recv [as 别名]
class ZQueue(object):
    def __init__(self, frontend_socket, backend_socket):
        self.queue = WorkerQueue()
        self.frontend = ZMQStream(frontend_socket)
        self.backend = ZMQStream(backend_socket)
        
        self.liveness = HEARTBEAT_LIVENESS
        self.heartbeat = HEARTBEAT_INTERVAL
        self.interval = INTERVAL_INIT
        self.loop = IOLoop.instance()
        self.hearbeats = 0

        self.time = self.interval * self.heartbeat
        self.heartbeat_at = time.time() + self.heartbeat * HEARTBEAT_LIVENESS        
        self.callback = None
        self.timed_out = False

        self.frontend.on_recv(self.handle_frontend)
        self.backend.on_recv(self.handle_backend)
        self.period = PeriodicCallback(self.purge,HEARTBEAT_INTERVAL*1000)
        self.period.start()

    def handle_frontend(self,msg):
        m = msg[:]
        if len(m) == 1:
            times_str('Received heartbeat')
            if self.timed_out:
                self.loop.add_timeout(time.time()+HEARTBEAT_INTERVAL, self.send_heartbeat)
                self.timed_out = False
                self.loop.remove_timeout(self.callback)
        elif len(m) == 3:
            times_str('Received: '+str(m))
            address, worker = self.queue.getLRU()
            worker.working = True
            m.insert(0,address)
            self.backend.send_multipart(m)
        self.heartbeat_at = time.time() + HEARTBEAT_INTERVAL * HEARTBEAT_LIVENESS

    def handle_backend(self,msg):
        m = msg[:]
        address = m[0]

        times_str('Backend Received: {}'.format(m))
        self.queue.ready(WorkerModel(address))
        self.backend.send_multipart([address,PPP_HEARTBEAT])

        mm = m[1:]
        if len(mm) == 1:
            if mm[0] == PPP_HEARTBEAT:
                self.hearbeats += 1
                times_str('Got hearbeat {}'.format(self.hearbeats))               
        else:
            times_str('Sending it back..')
            self.frontend.send_multipart(mm)

        if not self.queue.empty():
            self.frontend.on_recv(self.handle_frontend)

    def purge(self):
        self.queue.purge()
        if self.queue.empty():
            self.frontend.stop_on_recv()

    def send_heartbeat(self):
        if time.time() > self.heartbeat_at:
            self.time *= 2 if self.time < INTERVAL_MAX else 1
            times_str('Timed out.. Retrying in {} seconds..'.format(self.time))
            self.callback = self.loop.add_timeout(time.time()+self.time*1, self.send_heartbeat)
            self.timed_out = True
            return
        self.time = self.interval * self.heartbeat
        times_str('sending heartbeat..')
        self.frontend.send(PPP_HEARTBEAT)
        self.loop.add_timeout(time.time()+self.heartbeat, self.send_heartbeat)

    def run(self):
        try:
            self.frontend.send(PPP_READY)
            self.loop.start()
        except KeyboardInterrupt:
            times_str('ctrlc')