本文整理汇总了Python中zmq.eventloop.zmqstream.ZMQStream.stop_on_recv方法的典型用法代码示例。如果您正苦于以下问题:Python ZMQStream.stop_on_recv方法的具体用法?Python ZMQStream.stop_on_recv怎么用?Python ZMQStream.stop_on_recv使用的例子?那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类zmq.eventloop.zmqstream.ZMQStream
的用法示例。
在下文中一共展示了ZMQStream.stop_on_recv方法的14个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: ManagerControlled
# 需要导入模块: from zmq.eventloop.zmqstream import ZMQStream [as 别名]
# 或者: from zmq.eventloop.zmqstream.ZMQStream import stop_on_recv [as 别名]
class ManagerControlled(object):
def __init__(self, *args, **kwargs):
self.context = Context.instance()
self.loop = IOLoop.instance()
self.control_socket = self.context.socket(SUB)
self.control_socket.setsockopt(LINGER, 0) # discard unsent messages on close
self.control_socket.setsockopt(SUBSCRIBE, '')
self.control_socket.connect('tcp://{}:{}'.format(MANAGER_PUB_ADDRESS, MANAGER_PUB_PORT))
self.control_stream = ZMQStream(self.control_socket, self.loop)
self.control_stream.on_recv_stream(self.control_handler)
def control_handler(self, stream, message_list):
for message in message_list:
try:
notification, data = message.split()
except ValueError:
notification = message
if notification == NOTIFICATION_PROCESS_STOP:
self.stop()
def stop(self):
self.control_stream.stop_on_recv()
self.control_stream.close()
self.control_socket.close()
示例2: MonitorEvents
# 需要导入模块: from zmq.eventloop.zmqstream import ZMQStream [as 别名]
# 或者: from zmq.eventloop.zmqstream.ZMQStream import stop_on_recv [as 别名]
class MonitorEvents(SockJSConnection):
def _zmq_msg(self, msg):
#logging.debug(msg)
try:
msg_obj = json.loads(msg[0])
logging.debug(msg_obj)
if self.monitor != 'All':
if 'mon_id' in msg_obj and msg_obj['mon_id'] == self.monitor:
self.send(msg_obj)
else:
self.send(msg_obj)
except Exception as ex:
logging.error(ex)
def on_open(self, info):
logging.debug("Monitor ticker open: "+self.monitor)
zmq_socket = zmq.Context.instance().socket(zmq.SUB)
zmq_socket.connect(zmq_local_endpoint)
zmq_socket.setsockopt(zmq.SUBSCRIBE, '')
self.stream = ZMQStream(zmq_socket)
self.stream.on_recv(self._zmq_msg)
def on_close(self):
logging.debug("Monitor ticker close: "+self.monitor)
self.stream.stop_on_recv()
示例3: AsyncCircusClient
# 需要导入模块: from zmq.eventloop.zmqstream import ZMQStream [as 别名]
# 或者: from zmq.eventloop.zmqstream.ZMQStream import stop_on_recv [as 别名]
class AsyncCircusClient(object):
def __init__(self, context=None, endpoint=DEFAULT_ENDPOINT_DEALER,
timeout=5.0, ssh_server=None, ssh_keyfile=None):
self._init_context(context)
self.endpoint = endpoint
self._id = b(uuid.uuid4().hex)
self.socket = self.context.socket(zmq.DEALER)
self.socket.setsockopt(zmq.IDENTITY, self._id)
self.socket.setsockopt(zmq.LINGER, 0)
get_connection(self.socket, endpoint, ssh_server, ssh_keyfile)
self._timeout = timeout
self.timeout = timeout * 1000
self.stream = ZMQStream(self.socket, tornado.ioloop.IOLoop.instance())
def _init_context(self, context):
self.context = context or zmq.Context.instance()
def stop(self):
self.stream.stop_on_recv()
# only supported by libzmq >= 3
if hasattr(self.socket, 'disconnect'):
self.socket.disconnect(self.endpoint)
self.stream.close()
@tornado.gen.coroutine
def send_message(self, command, **props):
res = yield self.call(make_message(command, **props))
raise tornado.gen.Return(res)
@tornado.gen.coroutine
def call(self, cmd):
if isinstance(cmd, string_types):
raise DeprecationWarning('call() takes a mapping')
call_id = uuid.uuid4().hex
cmd['id'] = call_id
try:
cmd = json.dumps(cmd)
except ValueError as e:
raise CallError(str(e))
try:
yield tornado.gen.Task(self.stream.send, cmd)
except zmq.ZMQError as e:
raise CallError(str(e))
while True:
messages = yield tornado.gen.Task(self.stream.on_recv)
for message in messages:
try:
res = json.loads(message)
if res.get('id') != call_id:
# we got the wrong message
continue
raise tornado.gen.Return(res)
except ValueError as e:
raise CallError(str(e))
示例4: _Messenger
# 需要导入模块: from zmq.eventloop.zmqstream import ZMQStream [as 别名]
# 或者: from zmq.eventloop.zmqstream.ZMQStream import stop_on_recv [as 别名]
class _Messenger(object):
def __init__(self, in_sock, out_sock, context, io_loop=None):
self._context = context
self._io_loop = io_loop or IOLoop.instance()
self._create_socket(in_sock, out_sock)
self._in_stream = ZMQStream(self._in_socket, io_loop)
self._out_stream = ZMQStream(self._out_socket, io_loop)
self._callbacks = defaultdict(list)
def _create_socket(self, in_sock, out_sock):
raise NotImplementedError()
def start(self):
self._in_stream.on_recv(self._on_receive)
def stop(self):
self._in_stream.stop_on_recv()
# self._publish(CTRL_MSG_WORKER, None, CTRL_MSG_WORKER_QUIT_ACK)
#
def close(self):
self._in_stream.close()
self._in_socket.close()
self._out_stream.close()
self._out_socket.close()
def _on_receive(self, zmq_msg):
msg = CtrlMessage.deserialize(zmq_msg)
if msg.topic in self._callbacks:
for callback in self._callbacks[msg.topic]:
callback(msg)
# if msg.data == CTRL_MSG_WORKER_QUIT:
# self.stop()
def add_callback(self, topic, callback):
self._callbacks[topic].append(callback)
def remove_callback(self, topic, callback):
if topic in self._callbacks and callback in self._callbacks[topic]:
self._callbacks[topic].remove(callback)
def publish(self, topic, identity, data):
msg = CtrlMessage(topic, identity, data)
self._out_stream.send_multipart(msg.serialize())
示例5: AsynchronousStatsConsumer
# 需要导入模块: from zmq.eventloop.zmqstream import ZMQStream [as 别名]
# 或者: from zmq.eventloop.zmqstream.ZMQStream import stop_on_recv [as 别名]
class AsynchronousStatsConsumer(object):
def __init__(self, topics, loop, callback, context=None,
endpoint=DEFAULT_ENDPOINT_SUB, ssh_server=None, timeout=1.):
self.topics = topics
self.keep_context = context is not None
self.context = context or zmq.Context()
self.endpoint = endpoint
self.pubsub_socket = self.context.socket(zmq.SUB)
get_connection(self.pubsub_socket, self.endpoint, ssh_server)
for topic in self.topics:
self.pubsub_socket.setsockopt_string(zmq.SUBSCRIBE, topic)
self.stream = ZMQStream(self.pubsub_socket, loop)
self.stream.on_recv(self.process_message)
self.callback = callback
self.timeout = timeout
# Connection counter
self.count = 0
def __enter__(self):
return self
def __exit__(self, exc_type, exc_value, traceback):
""" On context manager exit, destroy the zmq context """
self.stop()
def process_message(self, msg):
topic, stat = msg
stat = stat.decode('utf-8')
topic = topic.decode('utf-8').split('.')
if len(topic) == 3:
__, watcher, subtopic = topic
self.callback(watcher, subtopic, json.loads(stat), self.endpoint)
elif len(topic) == 2:
__, watcher = topic
self.callback(watcher, None, json.loads(stat), self.endpoint)
def stop(self):
self.stream.stop_on_recv()
if self.keep_context:
return
try:
self.context.destroy(0)
except zmq.ZMQError as e:
if e.errno == errno.EINTR:
pass
else:
raise
示例6: LRUQueue
# 需要导入模块: from zmq.eventloop.zmqstream import ZMQStream [as 别名]
# 或者: from zmq.eventloop.zmqstream.ZMQStream import stop_on_recv [as 别名]
class LRUQueue(object):
"""LRUQueue class using ZMQStream/IOLoop for event dispatching"""
def __init__(self, backend_socket, frontend_socket):
self.available_workers = 0
self.workers = []
self.client_nbr = NBR_CLIENTS
self.backend = ZMQStream(backend_socket)
self.frontend = ZMQStream(frontend_socket)
self.backend.on_recv(self.handle_backend)
self.loop = IOLoop.instance()
def handle_backend(self, msg):
# Queue worker address for LRU routing
worker_addr, empty, client_addr = msg[:3]
assert self.available_workers < NBR_WORKERS
self.available_workers += 1
self.workers.append(worker_addr)
assert empty == b""
if client_addr != b"READY":
empty, reply = msg[3:]
assert empty == b""
self.frontend.send_multipart([client_addr, b'', reply])
self.client_nbr -= 1
if self.client_nbr == 0:
self.loop.add_timeout(time.time() + 1, self.loop.stop)
if self.available_workers == 1:
self.frontend.on_recv(self.handle_frontend)
def handle_frontend(self, msg):
client_addr, empty, request = msg
assert empty == b""
self.available_workers -= 1
worker_id = self.workers.pop()
self.backend.send_multipart([worker_id, b'', client_addr, b'', request])
if self.available_workers == 0:
self.frontend.stop_on_recv()
示例7: LRUQueue
# 需要导入模块: from zmq.eventloop.zmqstream import ZMQStream [as 别名]
# 或者: from zmq.eventloop.zmqstream.ZMQStream import stop_on_recv [as 别名]
class LRUQueue(object):
"""LRUQueue class using ZMQStream/IOLoop for event dispatching"""
def __init__(self, backend_socket, frontend_socket):
self.available_workers = 0
self.workers = []
self.client_nbr = NBR_CLIENTS
self.backend = ZMQStream(backend_socket)
self.frontend = ZMQStream(frontend_socket)
self.backend.on_recv(self.handle_backend)
self.loop = IOLoop.instance()
def handle_backend(self, msg):
# Queue worker address for LRU routing
worker_addr, empty, client_addr = msg[:3]
assert self.available_workers < NBR_WORKERS
# add worker back to the list of workers
self.available_workers += 1
self.workers.append(worker_addr)
# Second frame is empty
assert empty == b""
# Third frame is READY or else a client reply address
# If client reply, send rest back to frontend
if client_addr != b"READY":
empty, reply = msg[3:]
# Following frame is empty
assert empty == b""
self.frontend.send_multipart([client_addr, b'', reply])
self.client_nbr -= 1
if self.client_nbr == 0:
# Exit after N messages
self.loop.add_timeout(time.time()+1, self.loop.stop)
if self.available_workers == 1:
# on first recv, start accepting frontend messages
self.frontend.on_recv(self.handle_frontend)
def handle_frontend(self, msg):
# Now get next client request, route to LRU worker
# Client request is [address][empty][request]
client_addr, empty, request = msg
assert empty == b""
# Dequeue and drop the next worker address
self.available_workers -= 1
worker_id = self.workers.pop()
self.backend.send_multipart([worker_id, b'', client_addr, b'', request])
if self.available_workers == 0:
# stop receiving until workers become available again
self.frontend.stop_on_recv()
示例8: CloneServer
# 需要导入模块: from zmq.eventloop.zmqstream import ZMQStream [as 别名]
# 或者: from zmq.eventloop.zmqstream.ZMQStream import stop_on_recv [as 别名]
#.........这里部分代码省略.........
for key,kvmsg in self.kvmap.items():
self.flush_single(kvmsg)
def flush_single(self, kvmsg):
"""If key-value pair has expired, delete it and publish the fact
to listening clients."""
if kvmsg.get('ttl', 0) <= time.time():
kvmsg.body = ""
self.sequence += 1
kvmsg.sequence = self.sequence
kvmsg.send(self.publisher)
del self.kvmap[kvmsg.key]
logging.info("I: publishing delete=%d", self.sequence)
def send_hugz(self):
"""Send hugz to anyone listening on the publisher socket"""
kvmsg = KVMsg(self.sequence)
kvmsg.key = "HUGZ"
kvmsg.body = ""
kvmsg.send(self.publisher)
# ---------------------------------------------------------------------
# State change handlers
def become_master(self):
"""We're becoming master
The backup server applies its pending list to its own hash table,
and then starts to process state snapshot requests.
"""
self.master = True
self.slave = False
# stop receiving subscriber updates while we are master
self.subscriber.stop_on_recv()
# Apply pending list to own kvmap
while self.pending:
kvmsg = self.pending.pop(0)
self.sequence += 1
kvmsg.sequence = self.sequence
kvmsg.store(self.kvmap)
logging.info ("I: publishing pending=%d", self.sequence)
def become_slave(self):
"""We're becoming slave"""
# clear kvmap
self.kvmap = None
self.master = False
self.slave = True
self.subscriber.on_recv(self.handle_subscriber)
def handle_subscriber(self, msg):
"""Collect updates from peer (master)
We're always slave when we get these updates
"""
if self.master:
logging.warn("received subscriber message, but we are master %s", msg)
return
# Get state snapshot if necessary
if self.kvmap is None:
self.kvmap = {}
snapshot = self.ctx.socket(zmq.DEALER)
snapshot.linger = 0
snapshot.connect("tcp://localhost:%i" % self.peer)
示例9: ZmqWorker
# 需要导入模块: from zmq.eventloop.zmqstream import ZMQStream [as 别名]
# 或者: from zmq.eventloop.zmqstream.ZMQStream import stop_on_recv [as 别名]
class ZmqWorker(object, LoggingMixin):
"""
This is the ZMQ worker implementation.
The worker will register a :class:`ZMQStream` with the configured
:class:`zmq.Socket` and :class:`zmq.eventloop.ioloop.IOLoop` instance.
Upon `ZMQStream.on_recv` the configured `processors` will be executed
with the deserialized context and the result will be published through the
configured `zmq.socket`.
"""
def __init__(self, insocket, outsocket, mgmt, processing, log_handler,
log_level, io_loop=None):
"""
Initialize the `ZMQStream` with the `insocket` and `io_loop` and store
the `outsocket`.
`insocket` should be of the type `zmq.socket.PULL` `outsocket` should
be of the type `zmq.socket.PUB`
`mgmt` is an instance of `spyder.core.mgmt.ZmqMgmt` that handles
communication between master and worker processes.
"""
LoggingMixin.__init__(self, log_handler, log_level)
self._insocket = insocket
self._io_loop = io_loop or IOLoop.instance()
self._outsocket = outsocket
self._processing = processing
self._mgmt = mgmt
self._in_stream = ZMQStream(self._insocket, self._io_loop)
self._out_stream = ZMQStream(self._outsocket, self._io_loop)
def _quit(self, msg):
"""
The worker is quitting, stop receiving messages.
"""
if ZMQ_SPYDER_MGMT_WORKER_QUIT == msg.data:
self.stop()
def _receive(self, msg):
"""
We have a message!
`msg` is a serialized version of a `DataMessage`.
"""
message = DataMessage(msg)
try:
# this is the real work we want to do
curi = self._processing(message.curi)
message.curi = curi
except:
# catch any uncaught exception and only log it as CRITICAL
self._logger.critical(
"worker::Uncaught exception executing the worker for URL %s!" %
(message.curi.url,))
self._logger.critical("worker::%s" % (traceback.format_exc(),))
# finished, now send the result back to the master
self._out_stream.send_multipart(message.serialize())
def start(self):
"""
Start the worker.
"""
self._mgmt.add_callback(ZMQ_SPYDER_MGMT_WORKER, self._quit)
self._in_stream.on_recv(self._receive)
def stop(self):
"""
Stop the worker.
"""
# stop receiving
self._in_stream.stop_on_recv()
self._mgmt.remove_callback(ZMQ_SPYDER_MGMT_WORKER, self._quit)
# but work on anything we might already have
self._in_stream.flush()
self._out_stream.flush()
def close(self):
"""
Close all open sockets.
"""
self._in_stream.close()
self._insocket.close()
self._out_stream.close()
self._outsocket.close()
示例10: TornadoRPCService
# 需要导入模块: from zmq.eventloop.zmqstream import ZMQStream [as 别名]
# 或者: from zmq.eventloop.zmqstream.ZMQStream import stop_on_recv [as 别名]
class TornadoRPCService(RPCServiceBase): #{
""" An asynchronous RPC service that takes requests over a ROUTER socket.
Using Tornado compatible IOLoop and ZMQStream from PyZMQ.
"""
def __init__(self, context=None, ioloop=None, **kwargs): #{
"""
Parameters
==========
ioloop : IOLoop
An existing IOLoop instance, if not passed, zmq.IOLoop.instance()
will be used.
context : Context
An existing Context instance, if not passed, zmq.Context.instance()
will be used.
serializer : Serializer
An instance of a Serializer subclass that will be used to serialize
and deserialize args, kwargs and the result.
"""
assert context is None or isinstance(context, zmq.Context)
self.context = context if context is not None else zmq.Context.instance()
self.ioloop = IOLoop.instance() if ioloop is None else ioloop
self._is_started = False
super(TornadoRPCService, self).__init__(**kwargs)
#}
def _create_socket(self): #{
super(TornadoRPCService, self)._create_socket()
socket = self.context.socket(zmq.ROUTER)
self.socket = ZMQStream(socket, self.ioloop)
#}
def _handle_request(self, msg_list): #{
"""
Handle an incoming request.
The request is received as a multipart message:
[<id>..<id>, b'|', req_id, proc_name, <ser_args>, <ser_kwargs>, <ignore>]
First, the service sends back a notification that the message was
indeed received:
[<id>..<id>, b'|', req_id, b'ACK', service_id]
Next, the actual reply depends on if the call was successful or not:
[<id>..<id>, b'|', req_id, b'OK', <serialized result>]
[<id>..<id>, b'|', req_id, b'FAIL', <JSON dict of ename, evalue, traceback>]
Here the (ename, evalue, traceback) are utf-8 encoded unicode.
"""
req = self._parse_request(msg_list)
if req is None:
return
self._send_ack(req)
ignore = req['ignore']
try:
# raise any parsing errors here
if req['error']:
raise req['error']
# call procedure
res = req['proc'](*req['args'], **req['kwargs'])
except Exception:
not ignore and self._send_fail(req)
else:
def send_future_result(fut):
try: res = fut.result()
except: not ignore and self._send_fail(req)
else: not ignore and self._send_ok(req, res)
if isinstance(res, Future):
self.ioloop.add_future(res, send_future_result)
else:
not ignore and self._send_ok(req, res)
#}
def start(self): #{
""" Start the RPC service (non-blocking) """
assert self._is_started == False, "already started"
# register IOLoop callback
self._is_started = True
self.socket.on_recv(self._handle_request)
#}
def stop(self): #{
""" Stop the RPC service (non-blocking) """
# register IOLoop callback
self.socket.stop_on_recv()
self._is_started = False
#}
def serve(self): #{
""" Serve RPC requests (blocking) """
if not self._is_started:
self.start()
return self.ioloop.start()
示例11: Channel
# 需要导入模块: from zmq.eventloop.zmqstream import ZMQStream [as 别名]
# 或者: from zmq.eventloop.zmqstream.ZMQStream import stop_on_recv [as 别名]
class Channel(object):
"""Mother of all channels. Defines the interface.
Callbacks:
The callbacks will receive the channel as first parameter and
the message as second parameter. The error callback will get
the stream where the error occured as second parameter.
Attributes:
* stream_in, stream_out : the streams for eventlopp handling
* serializer : the serializer used
"""
def __init__(self, socket_in, socket_out, serializer):
self.stream_in = ZMQStream(socket_in)
self.stream_out = ZMQStream(socket_out)
self.serializer = serializer
self._cb_receive = None
self._cb_send = None
self._cb_error = None
self._chan_id = id(self)
return
def on_receive(self, callback):
"""Set callback to invoke when a message was received.
"""
self.stream_in.stop_on_recv()
self._cb_receive = callback
if callback:
self.stream_in.on_recv(self._on_recv)
return
def on_send(self, callback):
"""Set callback to invoke when a message was sent.
"""
self.stream_out.stop_on_send()
self._cb_send = callback
if callback:
self.stream_out.on_send(self._on_send)
return
def on_error(self, callback):
"""Set callback to invoke when an error event occured.
"""
self.stream_in.stop_on_err()
self.stream_out.stop_on_err()
self._cb_error = callback
if callback:
self.stream_in.on_err(self._on_err_in)
self.stream_out.on_err(self._on_err_out)
return
def send(self, message):
"""Send given message.
"""
m = self.serializer.serialize(message)
if self.serializer.multipart:
self.stream_out.send_multipart(m)
else:
self.stream_out.send(m)
return
def _on_recv(self, msg):
"""Helper interfacing w/ streams.
"""
if self.serializer.multipart:
msg = self.serializer.deserialize(msg)
else:
msg = self.serializer.deserialize(msg[0])
self._cb_receive(self, msg)
return
def _on_send(self, sent, _):
"""Helper interfacing w/ streams.
"""
msg = sent[0]
if self.serializer.multipart:
msg = self.serializer.deserialize(msg)
else:
msg = self.serializer.deserialize(msg[0])
self._cb_send(self, msg)
return
def _on_err_in(self):
self._cb_error(self, self.stream_in)
return
def _on_err_out(self):
self._cb_error(self, self.stream_out)
return
def __hash__(self):
return self._chan_id
def __eq__(self, other):
if not isinstance(other, self.__class__):
return False
return self._chan_id == other._chan_id
#.........这里部分代码省略.........
示例12: ZmqMgmt
# 需要导入模块: from zmq.eventloop.zmqstream import ZMQStream [as 别名]
# 或者: from zmq.eventloop.zmqstream.ZMQStream import stop_on_recv [as 别名]
class ZmqMgmt(object):
"""
A :class:`ZMQStream` object handling the management sockets.
"""
def __init__(self, subscriber, publisher, **kwargs):
"""
Initialize the management interface.
The `subscriber` socket is the socket used by the Master to send
commands to the workers. The publisher socket is used to send commands
to the Master.
You have to set the `zmq.SUBSCRIBE` socket option yourself!
"""
self._io_loop = kwargs.get('io_loop', IOLoop.instance())
self._subscriber = subscriber
self._in_stream = ZMQStream(self._subscriber, self._io_loop)
self._publisher = publisher
self._out_stream = ZMQStream(self._publisher, self._io_loop)
self._callbacks = dict()
def _receive(self, raw_msg):
"""
Main method for receiving management messages.
`message` is a multipart message where `message[0]` contains the topic,
`message[1]` is 0 and `message[1]` contains the actual message.
"""
msg = MgmtMessage(raw_msg)
if msg.topic in self._callbacks:
for callback in self._callbacks[msg.topic]:
if callable(callback):
callback(msg)
if ZMQ_SPYDER_MGMT_WORKER_QUIT == msg.data:
self.stop()
def start(self):
"""
Start the MGMT interface.
"""
self._in_stream.on_recv(self._receive)
def stop(self):
"""
Stop the MGMT interface.
"""
self._in_stream.stop_on_recv()
self.publish(topic=ZMQ_SPYDER_MGMT_WORKER, identity=None,
data=ZMQ_SPYDER_MGMT_WORKER_QUIT_ACK)
def close(self):
"""
Close all open sockets.
"""
self._in_stream.close()
self._subscriber.close()
self._out_stream.close()
self._publisher.close()
def add_callback(self, topic, callback):
"""
Add a callback to the specified topic.
"""
if not callable(callback):
raise ValueError('callback must be callable')
if topic not in self._callbacks:
self._callbacks[topic] = []
self._callbacks[topic].append(callback)
def remove_callback(self, topic, callback):
"""
Remove a callback from the specified topic.
"""
if topic in self._callbacks and callback in self._callbacks[topic]:
self._callbacks[topic].remove(callback)
def publish(self, topic=None, identity=None, data=None):
"""
Publish a message to the intended audience.
"""
assert topic is not None
assert data is not None
msg = MgmtMessage(topic=topic, identity=identity, data=data)
self._out_stream.send_multipart(msg.serialize())
示例13: PubSub
# 需要导入模块: from zmq.eventloop.zmqstream import ZMQStream [as 别名]
# 或者: from zmq.eventloop.zmqstream.ZMQStream import stop_on_recv [as 别名]
#.........这里部分代码省略.........
else:
zmq_pub_port = options.zmq_pub_port
self.zmq_pub_port = zmq_pub_port
publish_socket.bind(
"tcp://%s:%s" % (options.zmq_pub_listen, str(self.zmq_pub_port))
)
# wrap pub socket into ZeroMQ stream
self.pub_stream = ZMQStream(publish_socket)
# create SUB socket listening to all events from all app instances
subscribe_socket = self.zmq_context.socket(zmq.SUB)
if self.zmq_pub_sub_proxy:
# application started with XPUB/XSUB proxy
self.zmq_xpub = options.zmq_xpub
subscribe_socket.connect(self.zmq_xpub)
else:
# application started without XPUB/XSUB proxy
self.zmq_sub_address = options.zmq_sub_address
for address in self.zmq_sub_address:
subscribe_socket.connect(address)
subscribe_socket.setsockopt_string(
zmq.SUBSCRIBE,
six.u(CONTROL_CHANNEL)
)
subscribe_socket.setsockopt_string(
zmq.SUBSCRIBE, six.u(ADMIN_CHANNEL)
)
def listen_socket():
# wrap sub socket into ZeroMQ stream and set its on_recv callback
self.sub_stream = ZMQStream(subscribe_socket)
self.sub_stream.on_recv(self.dispatch_published_message)
tornado.ioloop.IOLoop.instance().add_callback(
listen_socket
)
if self.zmq_pub_sub_proxy:
logger.info(
"ZeroMQ XPUB: {0}, XSUB: {1}".format(self.zmq_xpub, self.zmq_xsub)
)
else:
logger.info(
"ZeroMQ PUB - {0}; subscribed to {1}".format(self.zmq_pub_port, self.zmq_sub_address)
)
def publish(self, channel, message, method=None):
"""
Publish message into channel of stream.
"""
method = method or self.DEFAULT_PUBLISH_METHOD
message["message_type"] = method
message = json_encode(message)
to_publish = [utf8(channel), utf8(message)]
self.pub_stream.send_multipart(to_publish)
@coroutine
def dispatch_published_message(self, multipart_message):
"""
Got message, decide what is it and dispatch into right
application handler.
"""
channel = multipart_message[0]
if six.PY3:
channel = channel.decode()
message_data = json_decode(multipart_message[1])
if channel == CONTROL_CHANNEL:
yield self.handle_control_message(message_data)
elif channel == ADMIN_CHANNEL:
yield self.handle_admin_message(message_data)
else:
yield self.handle_channel_message(channel, message_data)
def subscribe_key(self, subscription_key):
self.sub_stream.setsockopt_string(
zmq.SUBSCRIBE, six.u(subscription_key)
)
def unsubscribe_key(self, subscription_key):
self.sub_stream.setsockopt_string(
zmq.UNSUBSCRIBE, six.u(subscription_key)
)
def clean(self):
"""
Properly close ZeroMQ sockets.
"""
if hasattr(self, 'pub_stream') and self.pub_stream:
self.pub_stream.close()
if hasattr(self, 'sub_stream') and self.sub_stream:
self.sub_stream.stop_on_recv()
self.sub_stream.close()
示例14: ZQueue
# 需要导入模块: from zmq.eventloop.zmqstream import ZMQStream [as 别名]
# 或者: from zmq.eventloop.zmqstream.ZMQStream import stop_on_recv [as 别名]
class ZQueue(object):
def __init__(self, frontend_socket, backend_socket):
self.queue = WorkerQueue()
self.frontend = ZMQStream(frontend_socket)
self.backend = ZMQStream(backend_socket)
self.liveness = HEARTBEAT_LIVENESS
self.heartbeat = HEARTBEAT_INTERVAL
self.interval = INTERVAL_INIT
self.loop = IOLoop.instance()
self.hearbeats = 0
self.time = self.interval * self.heartbeat
self.heartbeat_at = time.time() + self.heartbeat * HEARTBEAT_LIVENESS
self.callback = None
self.timed_out = False
self.frontend.on_recv(self.handle_frontend)
self.backend.on_recv(self.handle_backend)
self.period = PeriodicCallback(self.purge,HEARTBEAT_INTERVAL*1000)
self.period.start()
def handle_frontend(self,msg):
m = msg[:]
if len(m) == 1:
times_str('Received heartbeat')
if self.timed_out:
self.loop.add_timeout(time.time()+HEARTBEAT_INTERVAL, self.send_heartbeat)
self.timed_out = False
self.loop.remove_timeout(self.callback)
elif len(m) == 3:
times_str('Received: '+str(m))
address, worker = self.queue.getLRU()
worker.working = True
m.insert(0,address)
self.backend.send_multipart(m)
self.heartbeat_at = time.time() + HEARTBEAT_INTERVAL * HEARTBEAT_LIVENESS
def handle_backend(self,msg):
m = msg[:]
address = m[0]
times_str('Backend Received: {}'.format(m))
self.queue.ready(WorkerModel(address))
self.backend.send_multipart([address,PPP_HEARTBEAT])
mm = m[1:]
if len(mm) == 1:
if mm[0] == PPP_HEARTBEAT:
self.hearbeats += 1
times_str('Got hearbeat {}'.format(self.hearbeats))
else:
times_str('Sending it back..')
self.frontend.send_multipart(mm)
if not self.queue.empty():
self.frontend.on_recv(self.handle_frontend)
def purge(self):
self.queue.purge()
if self.queue.empty():
self.frontend.stop_on_recv()
def send_heartbeat(self):
if time.time() > self.heartbeat_at:
self.time *= 2 if self.time < INTERVAL_MAX else 1
times_str('Timed out.. Retrying in {} seconds..'.format(self.time))
self.callback = self.loop.add_timeout(time.time()+self.time*1, self.send_heartbeat)
self.timed_out = True
return
self.time = self.interval * self.heartbeat
times_str('sending heartbeat..')
self.frontend.send(PPP_HEARTBEAT)
self.loop.add_timeout(time.time()+self.heartbeat, self.send_heartbeat)
def run(self):
try:
self.frontend.send(PPP_READY)
self.loop.start()
except KeyboardInterrupt:
times_str('ctrlc')