本文整理汇总了Python中zmq.eventloop.ioloop.PeriodicCallback.stop方法的典型用法代码示例。如果您正苦于以下问题:Python PeriodicCallback.stop方法的具体用法?Python PeriodicCallback.stop怎么用?Python PeriodicCallback.stop使用的例子?那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类zmq.eventloop.ioloop.PeriodicCallback
的用法示例。
在下文中一共展示了PeriodicCallback.stop方法的15个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: WorkerRep
# 需要导入模块: from zmq.eventloop.ioloop import PeriodicCallback [as 别名]
# 或者: from zmq.eventloop.ioloop.PeriodicCallback import stop [as 别名]
class WorkerRep(object):
"""Helper class to represent a worker in the broker.
Instances of this class are used to track the state of the attached worker
and carry the timers for incomming and outgoing heartbeats.
:type wid: str
:param wid: the worker id.
:param service: service this worker serves
:type service: str
:param stream: the ZMQStream used to send messages
:type stream: ZMQStream
"""
def __init__(self, wid, service, stream):
self.id = wid
self.service = service
self.multicasts = []
self.curr_liveness = HB_LIVENESS
self.stream = stream
self.last_hb = 0
self.hb_out_timer = PeriodicCallback(self.send_hb, HB_INTERVAL)
self.hb_out_timer.start()
return
def send_hb(self):
"""Called on every HB_INTERVAL.
Decrements the current liveness by one.
Sends heartbeat to worker.
"""
self.curr_liveness -= 1
msg = [ self.id, b'', MDP_WORKER_VERSION, b'\x05' ]
self.stream.send_multipart(msg)
return
def on_heartbeat(self):
"""Called when a heartbeat message from the worker was received.
Sets current liveness to HB_LIVENESS.
"""
self.curr_liveness = HB_LIVENESS
return
def is_alive(self):
"""Returns True when the worker is considered alive.
"""
return self.curr_liveness > 0
def shutdown(self):
"""Cleanup worker.
Stops timer.
"""
self.hb_out_timer.stop()
self.hb_out_timer = None
self.stream = None
return
示例2: DeviceRep
# 需要导入模块: from zmq.eventloop.ioloop import PeriodicCallback [as 别名]
# 或者: from zmq.eventloop.ioloop.PeriodicCallback import stop [as 别名]
class DeviceRep(object):
"""
Helper class to represent a device to a worker
"""
def __init__(self, device_id, state='unknown'):
self.id = device_id
self.state = state
self.curr_liveness = CLIENT_HB_LIVENESS
self.hb_timer = PeriodicCallback(self.heartbeat, CLIENT_HB_INTERVAL)
self.hb_timer.start()
return
def heartbeat(self):
if self.curr_liveness > 0:
self.curr_liveness -= 1
if self.curr_liveness == 0:
self.state = 'dead'
return
def on_message_received(self):
self.curr_liveness = CLIENT_HB_LIVENESS
return
def is_alive(self):
return self.curr_liveness > 0
def get_state(self):
return self.state
def shutdown(self):
self.hb_timer.stop()
self.hb_timer = None
示例3: TaskState
# 需要导入模块: from zmq.eventloop.ioloop import PeriodicCallback [as 别名]
# 或者: from zmq.eventloop.ioloop.PeriodicCallback import stop [as 别名]
class TaskState (object):
""" Tracks task state (with help of watchdog) """
log = skytools.getLogger ('d:TaskState')
def __init__ (self, uid, name, info, ioloop, cc, xtx):
self.uid = uid
self.name = name
self.info = info
self.pidfile = info['config']['pidfile']
self.ioloop = ioloop
self.cc = cc
self.xtx = xtx
self.timer = None
self.timer_tick = 1
self.heartbeat = False
self.start_time = None
self.dead_since = None
def start (self):
self.start_time = time.time()
self.timer = PeriodicCallback (self.watchdog, self.timer_tick * 1000, self.ioloop)
self.timer.start()
def stop (self):
try:
self.log.info ('Signalling %s', self.name)
skytools.signal_pidfile (self.pidfile, signal.SIGINT)
except:
self.log.exception ('signal_pidfile failed: %s', self.pidfile)
def watchdog (self):
live = skytools.signal_pidfile (self.pidfile, 0)
if live:
self.log.debug ('%s is alive', self.name)
if self.heartbeat:
self.send_reply ('running')
else:
self.log.info ('%s is over', self.name)
self.dead_since = time.time()
self.timer.stop()
self.timer = None
self.send_reply ('stopped')
def ccpublish (self, msg):
assert isinstance (msg, TaskReplyMessage)
cmsg = self.xtx.create_cmsg (msg)
cmsg.send_to (self.cc)
def send_reply (self, status, feedback = {}):
msg = TaskReplyMessage(
req = 'task.reply.%s' % self.uid,
handler = self.info['task']['task_handler'],
task_id = self.info['task']['task_id'],
status = status,
feedback = feedback)
self.ccpublish (msg)
示例4: DemoApp
# 需要导入模块: from zmq.eventloop.ioloop import PeriodicCallback [as 别名]
# 或者: from zmq.eventloop.ioloop.PeriodicCallback import stop [as 别名]
class DemoApp(ZMQProcess):
def __init__ (self):
super(DemoApp, self).__init__()
def setup(self):
super(DemoApp, self).setup()
self.pub, self.pub_addr = self.stream(zmq.PUB, 'tcp://127.0.0.1:%(port)s', True)
self.sub, sub_addr = self.stream(zmq.SUB, self.pub_addr, False,
callback=DemoHandler())
self.heartbeat = PeriodicCallback(self.ping, 1000, self.loop)
def ping(self):
print 'SEND PING'
self.pub.send_multipart(['ping', json.dumps(['ping', time.time()])])
def local_run(self):
print 'START HEARTBEAT'
self.heartbeat.start()
def stop(self):
self.heartbeat.stop()
self.loop.stop()
示例5: TailWriter
# 需要导入模块: from zmq.eventloop.ioloop import PeriodicCallback [as 别名]
# 或者: from zmq.eventloop.ioloop.PeriodicCallback import stop [as 别名]
#.........这里部分代码省略.........
# initialise sockets for communication with workers
self.dealer_stream, self.dealer_url = self.init_comm (zmq.XREQ, 'inproc://workers-dealer', self.dealer_on_recv)
self.router_stream, self.router_url = self.init_comm (zmq.XREP, 'inproc://workers-router', self.router_on_recv)
self.launch_workers()
self.timer_maint = PeriodicCallback (self.do_maint, self.wparams['maint_period'] * 1000, self.ioloop)
self.timer_maint.start()
def init_comm (self, stype, url, cb):
""" Create socket, stream, etc for communication with workers. """
sock = self.zctx.socket (stype)
port = sock.bind_to_random_port (url)
curl = "%s:%d" % (url, port)
stream = CCStream (sock, self.ioloop)
stream.on_recv (cb)
return (stream, curl)
def launch_workers (self):
""" Create and start worker threads. """
nw = self.cf.getint ('worker-threads', 10)
for i in range (nw):
wname = "%s.worker-%i" % (self.hname, i)
self.log.info ("starting %s", wname)
w = TailWriter_Worker(
wname, self.xtx, self.zctx, self.ioloop,
self.dealer_url, self.router_url, self.wparams)
w.stat_inc = self.stat_inc # XXX
self.workers.append (w)
w.start()
def handle_msg (self, cmsg):
""" Got message from client, process it. """
data = cmsg.get_payload (self.xtx)
if not data: return
host = data['hostname']
fn = data['filename']
st_dev = data.get('st_dev')
st_ino = data.get('st_ino')
fi = (host, st_dev, st_ino, fn)
if fi in self.files:
fd = self.files[fi]
if fd.waddr: # already accepted ?
self.log.trace ("passing %r to %s", fn, fd.wname)
fd.queue.append (cmsg)
fd.send_to (self.router_stream)
else:
self.log.trace ("queueing %r", fn)
fd.queue.append (cmsg)
else:
fd = FileState (fi, 1)
self.files[fi] = fd
self.log.trace ("offering %r", fn)
self.dealer_stream.send_cmsg (cmsg)
def dealer_on_recv (self, zmsg):
""" Got reply from worker via "dealer" connection """
self.log.warning ("reply via dealer: %s", zmsg)
def router_on_recv (self, zmsg):
""" Got reply from worker via "router" connection """
cmsg = CCMessage (zmsg)
data = cmsg.get_payload (self.xtx)
fi = (data['d_hostname'], data['d_st_dev'], data['d_st_ino'], data['d_filename'])
fd = self.files[fi]
if fd.waddr is None:
fd.waddr = zmsg[0]
fd.wname = data['worker']
else:
assert fd.waddr == zmsg[0] and fd.wname == data['worker']
fd.atime = time.time()
fd.count -= 1
assert fd.count >= 0
def do_maint (self):
""" Check & flush queues; drop inactive files. """
self.log.trace ('cleanup')
now = time.time()
zombies = []
for k, fd in self.files.iteritems():
if fd.queue and fd.waddr:
self.log.trace ("passing %r to %s", fd.ident, fd.wname)
fd.send_to (self.router_stream)
if (fd.count == 0) and (now - fd.atime > 2 * CLOSE_DELAY): # you'd better use msg for this
self.log.debug ("forgetting %r", fd.ident)
zombies.append(k)
for k in zombies:
self.files.pop(k)
def stop (self):
""" Signal workers to shut down. """
super(TailWriter, self).stop()
self.log.info ('stopping')
self.timer_maint.stop()
for w in self.workers:
self.log.info ("signalling %s", w.name)
w.stop()
示例6: MDPWorker
# 需要导入模块: from zmq.eventloop.ioloop import PeriodicCallback [as 别名]
# 或者: from zmq.eventloop.ioloop.PeriodicCallback import stop [as 别名]
class MDPWorker(object):
"""Class for the MDP worker side.
Thin encapsulation of a zmq.XREQ socket.
Provides a send method with optional timeout parameter.
Will use a timeout to indicate a broker failure.
"""
_proto_version = b'MDPW01'
# TODO: integrate that into API
HB_INTERVAL = 1000 # in milliseconds
HB_LIVENESS = 3 # HBs to miss before connection counts as dead
def __init__(self, context, endpoint, service):
"""Initialize the MDPWorker.
context is the zmq context to create the socket from.
service is a byte-string with the service name.
"""
self.context = context
self.endpoint = endpoint
self.service = service
self.stream = None
self._tmo = None
self.need_handshake = True
self.ticker = None
self._delayed_cb = None
self._create_stream()
return
def _create_stream(self):
"""Helper to create the socket and the stream.
"""
socket = self.context.socket(zmq.XREQ)
ioloop = IOLoop.instance()
self.stream = ZMQStream(socket, ioloop)
self.stream.on_recv(self._on_message)
self.stream.socket.setsockopt(zmq.LINGER, 0)
self.stream.connect(self.endpoint)
self.ticker = PeriodicCallback(self._tick, self.HB_INTERVAL)
self._send_ready()
self.ticker.start()
return
def _send_ready(self):
"""Helper method to prepare and send the workers READY message.
"""
ready_msg = [ b'', self._proto_version, chr(1), self.service ]
self.stream.send_multipart(ready_msg)
self.curr_liveness = self.HB_LIVENESS
return
def _tick(self):
"""Method called every HB_INTERVAL milliseconds.
"""
self.curr_liveness -= 1
## print '%.3f tick - %d' % (time.time(), self.curr_liveness)
self.send_hb()
if self.curr_liveness >= 0:
return
print '%.3f lost connection' % time.time()
# ouch, connection seems to be dead
self.shutdown()
# try to recreate it
self._delayed_cb = DelayedCallback(self._create_stream, 5000)
self._delayed_cb.start()
return
def send_hb(self):
"""Construct and send HB message to broker.
"""
msg = [ b'', self._proto_version, chr(4) ]
self.stream.send_multipart(msg)
return
def shutdown(self):
"""Method to deactivate the worker connection completely.
Will delete the stream and the underlying socket.
"""
if self.ticker:
self.ticker.stop()
self.ticker = None
if not self.stream:
return
self.stream.socket.close()
self.stream.close()
self.stream = None
self.timed_out = False
self.need_handshake = True
self.connected = False
return
def reply(self, msg):
"""Send the given message.
msg can either be a byte-string or a list of byte-strings.
#.........这里部分代码省略.........
示例7: Worker
# 需要导入模块: from zmq.eventloop.ioloop import PeriodicCallback [as 别名]
# 或者: from zmq.eventloop.ioloop.PeriodicCallback import stop [as 别名]
class Worker(object):
"""Class for the MDP worker side.
Thin encapsulation of a zmq.DEALER socket.
Provides a send method with optional timeout parameter.
Will use a timeout to indicate a broker failure.
"""
max_forks = 10
ipc = 'ipc:///tmp/zmq-rpc-'+str(uuid4())
HB_INTERVAL = 1000 # in milliseconds
HB_LIVENESS = 3 # HBs to miss before connection counts as dead
def __init__(self, context, endpoint, service, multicasts=()):
"""Initialize the MDPWorker.
:param context: is the zmq context to create the socket from
:type context: zmq.Context
:param service: service name - you can put hostname here
:type service: str
:param multicasts: list of groups to subscribe
:type multicasts: list
"""
self.context = context
self.endpoint = endpoint
self.service = service.encode('utf-8') # convert to byte-string - required in python 3
self.multicasts = [m.encode('utf-8') for m in multicasts] # convert to byte-string
self.stream = None
self._tmo = None
self.need_handshake = True
self.ticker = None
self._delayed_cb = None
self._create_stream()
self.forks = []
self.curr_liveness = self.HB_LIVENESS
socket = self.context.socket(zmq.ROUTER)
socket.bind(self.ipc)
self.stream_w = ZMQStream(socket)
self.stream_w.on_recv(self._on_fork_response)
self.reply_socket = None
return
def _create_stream(self):
"""Helper to create the socket and the stream.
"""
self.on_log_event('broker.connect', 'Trying to connect do broker')
socket = self.context.socket(zmq.DEALER)
ioloop = IOLoop.instance()
self.stream = ZMQStream(socket, ioloop)
self.stream.on_recv(self._on_message)
self.stream.socket.setsockopt(zmq.LINGER, 0)
self.stream.connect(self.endpoint)
self.ticker = PeriodicCallback(self._tick, self.HB_INTERVAL)
self._send_ready()
for m in self.multicasts:
self._register_worker_to_multicast(m)
self.ticker.start()
return
def _tick(self):
"""Method called every HB_INTERVAL milliseconds.
"""
self.curr_liveness -= 1
self.send_hb()
if self.curr_liveness >= 0:
return
# ouch, connection seems to be dead
self.on_log_event('broker.timeout', 'Connection to broker timeouted, disconnecting')
self.shutdown(False)
# try to recreate it
self._delayed_cb = DelayedCallback(self._create_stream, 5000)
self._delayed_cb.start()
return
def send_hb(self):
"""Construct and send HB message to broker.
"""
msg = [b'', MDP_WORKER_VERSION, b'\x05']
self.stream.send_multipart(msg)
return
def shutdown(self, final=True):
"""Method to deactivate the worker connection completely.
Will delete the stream and the underlying socket.
:param final: if shutdown is final and we want to close all sockets
:type final: bool
"""
if self.ticker:
self.ticker.stop()
self.ticker = None
if not self.stream:
return
self.stream.on_recv(None)
#.........这里部分代码省略.........
示例8: MQRep
# 需要导入模块: from zmq.eventloop.ioloop import PeriodicCallback [as 别名]
# 或者: from zmq.eventloop.ioloop.PeriodicCallback import stop [as 别名]
class MQRep(object):
"""Class for the MDP worker side.
Thin encapsulation of a zmq.DEALER socket.
Provides a send method with optional timeout parameter.
Will use a timeout to indicate a broker failure.
"""
_proto_version = b'MDPW01'
# TODO: integrate that into API
HB_INTERVAL = 1000 # in milliseconds
HB_LIVENESS = 3 # HBs to miss before connection counts as dead
def __init__(self, context, service):
"""Initialize the MDPWorker.
context is the zmq context to create the socket from.
service is a byte-string with the service name.
"""
if DEBUG:
print("MQRep > __init__")
cfg = Loader('mq').load()
config = dict(cfg[1])
if config['ip'].strip() == "*":
config['ip'] = get_ip()
self.endpoint = "tcp://{0}:{1}".format(config['ip'], config['req_rep_port'])
self.context = context
self.service = service
self.stream = None
self._tmo = None
self.need_handshake = True
self.ticker = None
self._delayed_cb = None
self._create_stream()
### patch fritz
self._reconnect_in_progress = False
### end patch fritz
return
def _create_stream(self):
"""Helper to create the socket and the stream.
"""
if DEBUG:
print("MQRep > _create_stream")
socket = ZmqSocket(self.context, zmq.DEALER)
ioloop = IOLoop.instance()
self.stream = ZMQStream(socket, ioloop)
self.stream.on_recv(self._on_mpd_message)
self.stream.socket.setsockopt(zmq.LINGER, 0)
self.stream.connect(self.endpoint)
if self.ticker != None:
if DEBUG:
print("MQRep > _create_stream - stop ticker")
self.ticker.stop()
self.ticker = PeriodicCallback(self._tick, self.HB_INTERVAL)
self._send_ready()
self.ticker.start()
return
def _send_ready(self):
"""Helper method to prepare and send the workers READY message.
"""
if DEBUG:
print("MQREP > _send_ready")
ready_msg = [ b'', self._proto_version, b'\x01', self.service ]
self.stream.send_multipart(ready_msg)
self.curr_liveness = self.HB_LIVENESS
if DEBUG:
print("MQREP > _send_ready > curr_liveness <= {0}".format(self.HB_LIVENESS))
return
def _tick(self):
"""Method called every HB_INTERVAL milliseconds.
"""
if DEBUG:
print("MQREP > _tick")
self.curr_liveness -= 1
if DEBUG:
print('MQREP > _tick - {0} tick = {1}'.format(time.time(), self.curr_liveness))
self.send_hb()
if self.curr_liveness >= 0:
return
if DEBUG:
print('MQREP > _tick - {0} lost connection'.format(time.time()))
# ouch, connection seems to be dead
self.shutdown()
# try to recreate it
self._delayed_cb = DelayedCallback(self._create_stream, self.HB_INTERVAL)
self._delayed_cb.start()
return
def send_hb(self):
"""Construct and send HB message to broker.
"""
msg = [ b'', self._proto_version, b'\x04' ]
self.stream.send_multipart(msg)
#.........这里部分代码省略.........
示例9: Master
# 需要导入模块: from zmq.eventloop.ioloop import PeriodicCallback [as 别名]
# 或者: from zmq.eventloop.ioloop.PeriodicCallback import stop [as 别名]
class Master(object):
def __init__(self, frontier,
data_in_sock='ipc:///tmp/robot-data-w2m.sock',
data_out_sock='ipc:///tmp/robot-data-m2w.sock',
msg_in_sock='ipc:///tmp/robot-msg-w2m.sock',
msg_out_sock='ipc:///tmp/robot-msg-m2w.sock',
io_loop=None):
self.identity = 'master:%s:%s' % (socket.gethostname(), os.getpid())
context = zmq.Context()
self._io_loop = io_loop or IOLoop.instance()
self._in_socket = context.socket(zmq.SUB)
self._in_socket.setsockopt(zmq.SUBSCRIBE, '')
self._in_socket.bind(data_in_sock)
self._in_stream = ZMQStream(self._in_socket, io_loop)
self._out_socket = context.socket(zmq.PUSH)
self._out_socket.bind(data_out_sock)
self._out_stream = ZMQStream(self._out_socket, io_loop)
self._online_workers = set()
self._running = False
self._updater = PeriodicCallback(self._send_next, 100, io_loop=io_loop)
self._reloader = PeriodicCallback(self.reload, 1000, io_loop=io_loop)
self.frontier = frontier
self.messenger = ServerMessenger(msg_in_sock, msg_out_sock,
context, io_loop)
def start(self):
logging.info('[%s] starting', self.identity)
self.messenger.add_callback(CTRL_MSG_WORKER, self._on_worker_msg)
self.messenger.start()
self._in_stream.on_recv(self._on_receive_processed)
self._updater.start()
self._reloader.start()
self._running = True
def stop(self):
self._running = False
self._reloader.stop()
self._updater.stop()
self.messenger.stop()
# self.messenger.publish(CTRL_MSG_WORKER, self.identity,
# CTRL_MSG_WORKER_QUIT)
def close(self):
self._in_stream.close()
self._in_socket.close()
self._out_stream.close()
self._out_socket.close()
self.messenger.close()
def reload(self):
pass
def _on_worker_msg(self, msg):
if msg.data == CTRL_MSG_WORKER_ONLINE:
self._online_workers.add(msg.identity)
logging.info('[%s] append [%s]', self.identity, msg.identity)
self._send_next()
# if msg.data == CTRL_MSG_WORKER_QUIT_ACK:
# if msg.identity in self._online_workers:
# self._online_workers.remove(msg.identity)
def _send_next(self):
if not self._running:
return
worker_num = len(self._online_workers)
if self._running and worker_num > 0:
while self._out_stream._send_queue.qsize() < worker_num * 4:
request = self.frontier.get_next_request()
if not request:
break
msg = RequestMessage(self.identity, request)
self._out_stream.send_multipart(msg.serialize())
logging.debug('[%s] send request(%s)',
self.identity, request.url)
self.frontier.reload_request(request)
def _on_receive_processed(self, zmq_msg):
msg = ResponseMessage.deserialize(zmq_msg)
request = msg.response.request
logging.debug('[%s] receive response(%s)', self.identity, request.url)
self._send_next()
示例10: DeviceServiceManager
# 需要导入模块: from zmq.eventloop.ioloop import PeriodicCallback [as 别名]
# 或者: from zmq.eventloop.ioloop.PeriodicCallback import stop [as 别名]
class DeviceServiceManager(MDPWorker):
_connected_devices = {}
def __init__(self, context, endpoint, service):
"""Overridden initializer for MDPWorker.
Adds the device_timer to manage connected devices
"""
session = Session()
session.query(Device).update({Device.connected: False})
session.commit()
self.device_timer = None
super(DeviceServiceManager, self).__init__(context, endpoint, service)
return
def _create_stream(self):
"""Overidden _create_stream for MDPWorker
Adds the device_timer to manage connected devices
"""
self.device_timer = PeriodicCallback(self.device_watcher,
CLIENT_HB_INTERVAL)
self.device_timer.start()
super(DeviceServiceManager, self)._create_stream()
return
def shutdown(self):
"""Overidden shutdown for MDPWorker
Adds the device_timer to manage connected devices
"""
self.device_timer.stop()
self.device_timer = None
super(DeviceServiceManager, self).shutdown()
return
def device_watcher(self):
for device in self._connected_devices.values():
if not device.is_alive():
session = Session()
session.query(Device).filter_by(id=device.id) \
.update({Device.connected: False})
session.commit()
device.shutdown()
del self._connected_devices[device.id]
return
def on_heartbeat(self, did):
if did in self._connected_devices:
if self._connected_devices[did].is_alive():
self._connected_devices[did].on_message_received()
return self._connected_devices[did].get_state()
else:
session = Session()
device = session.query(Device).filter_by(id=did).first()
if not device:
session.close()
# signals the device to send a create message
return 'unknown'
else:
self._connected_devices[did] = DeviceRep(did,
state='connected')
device.connected = True
session.commit()
return 'connected'
return
def on_create(self, did, msg):
imported_device = pickle.loads(msg[0])
device = Device(imported_device['id'],
version=imported_device['version'])
device.connected = True
device.user_configured = False
device.database_service = imported_device['database_service']
device.device_service = imported_device['device_service']
device.grainbin_service = imported_device['grainbin_service']
device.grainbin_count = imported_device['grainbin_count']
session = Session()
session.add(device)
if device.grainbin_service:
grainbins = []
for x in range(device.grainbin_count):
id = device.id + '.' + str(x).zfill(2)
grainbin = Grainbin(id, device.id, x)
grainbins.append(grainbin)
session.add_all(grainbins)
session.commit()
session.close()
return "added"
def on_remove(self, did, msg):
if did in self._connected_devices:
self._connected_devices[did].shutdown()
del self._connected_devices[did]
#.........这里部分代码省略.........
示例11: Echo
# 需要导入模块: from zmq.eventloop.ioloop import PeriodicCallback [as 别名]
# 或者: from zmq.eventloop.ioloop.PeriodicCallback import stop [as 别名]
#.........这里部分代码省略.........
for url in self.cf.getlist ("ping-remotes", ""):
sock = self._make_socket (url)
self.stream[url] = CCStream (sock, ccscript.ioloop, qmaxsize = self.zmq_hwm)
self.stream[url].on_recv (self.on_recv)
self.echoes[url] = EchoState (url)
self.log.debug ("will ping %s", url)
self.timer = PeriodicCallback (self.ping, self.ping_tick * 1000, self.ioloop)
self.timer.start()
def _make_socket (self, url):
""" Create socket for pinging remote CC. """
sock = self.zctx.socket (zmq.XREQ)
sock.setsockopt (zmq.HWM, self.zmq_hwm)
sock.setsockopt (zmq.LINGER, self.zmq_linger)
sock.connect (url)
return sock
def on_recv (self, zmsg):
""" Got reply from a remote CC, process it. """
try:
self.log.trace ("%r", zmsg)
cmsg = CCMessage (zmsg)
req = cmsg.get_dest()
if req == "echo.response":
self.process_response (cmsg)
else:
self.log.warn ("unknown msg: %s", req)
except:
self.log.exception ("crashed, dropping msg")
def handle_msg (self, cmsg):
""" Got a message, process it. """
self.log.trace ("%r", cmsg)
req = cmsg.get_dest()
if req == "echo.request":
self.process_request (cmsg)
else:
self.log.warn ("unknown msg: %s", req)
def process_request (self, cmsg):
""" Ping received, respond with pong. """
msg = cmsg.get_payload (self.xtx)
if not msg: return
rep = EchoResponseMessage(
orig_hostname = msg['hostname'],
orig_target = msg['target'],
orig_time = msg['time'])
rcm = self.xtx.create_cmsg (rep)
rcm.take_route (cmsg)
rcm.send_to (self.cclocal)
def process_response (self, cmsg):
""" Pong received, evaluate it. """
msg = cmsg.get_payload (self.xtx)
if not msg: return
url = msg.orig_target
if url not in self.echoes:
self.log.warn ("unknown pong: %s", url)
return
echo = self.echoes[url]
echo.update_pong (msg)
rtt = echo.time_pong - msg.orig_time
if msg.orig_time == echo.time_ping:
self.log.trace ("echo time: %f s (%s)", rtt, url)
elif rtt <= 5 * self.ping_tick:
self.log.debug ("late pong: %f s (%s)", rtt, url)
else:
self.log.info ("too late pong: %f s (%s)", rtt, url)
def send_request (self, url):
""" Send ping to remote CC. """
msg = EchoRequestMessage(
target = url)
cmsg = self.xtx.create_cmsg (msg)
self.stream[url].send_cmsg (cmsg)
self.echoes[url].update_ping (msg)
self.log.trace ("%r", msg)
def ping (self):
""" Echo requesting and monitoring. """
self.log.trace ("")
for url in self.stream:
echo = self.echoes[url]
if echo.time_ping - echo.time_pong > 5 * self.ping_tick:
self.log.warn ("no pong from %s for %f s", url, echo.time_ping - echo.time_pong)
self.send_request (url)
def stop (self):
super(Echo, self).stop()
self.log.info ("stopping")
self.timer.stop()
示例12: DeviceConnection
# 需要导入模块: from zmq.eventloop.ioloop import PeriodicCallback [as 别名]
# 或者: from zmq.eventloop.ioloop.PeriodicCallback import stop [as 别名]
class DeviceConnection(object):
SERVICE_NAME = 'device' # service to connect to
TIMEOUT = 5 # time to wait for answer in seconds
# Number of connections to try before restarting when in incorrect state
CONNECTION_ATTEMPS = 3
HB_INTERVAL = 1000 * 10 # in milliseconds
def __init__(self, context, device_id, address):
self.context = context
self.device_id = device_id
self.address = address
self.ticker = None
self.updater = None
self.socket = None
self.can_send = False
self._restart()
self.heartbeat()
return
def _restart(self):
self.shutdown()
self.socket = self.context.socket(zmq.REQ)
self.socket.setsockopt(zmq.LINGER, 0)
self.socket.connect(self.address)
self.can_send = True
self.connection_attempts = 0
self.device_state = 'unknown'
self.ticker = PeriodicCallback(self.heartbeat,
DeviceConnection.HB_INTERVAL)
self.ticker.start()
# delay = seconds_till_next('hour', duration=1) + 2 # 2 second buffer
delay = 60 # send update one minute after startup
self.updater = DelayedCallback(self.update, delay * 1000)
self.updater.start()
return
def shutdown(self):
if self.ticker:
self.ticker.stop()
self.ticker = None
if self.updater:
self.updater.stop()
if self.socket:
self.socket.close()
self.socket = None
self.can_send = False
return
def send(self, message):
if not self.can_send:
self.connection_attempts += 1
logger.error("DeviceConnection is not in state to send")
return None
else:
self.connection_attempts = 0
self.can_send = False
logger.debug("DeviceConnection sending {0}".format(message))
reply = mdp_request(self.socket,
DeviceConnection.SERVICE_NAME,
message,
DeviceConnection.TIMEOUT)
if reply:
logger.debug("DeviceConnection reply received: {0}".format(reply))
self.can_send = True
return reply
else:
# Timeout! Will be restarted at next heartbeat
logger.warn("DeviceConnection timeout. Will be restarted at next heartbeat")
self.connection_attempts = DeviceConnection.CONNECTION_ATTEMPS
return None
def heartbeat(self):
if self.connection_attempts >= DeviceConnection.CONNECTION_ATTEMPS:
logger.warn("DeviceConnection attempts max reached. Restarting connection.")
self._restart()
message = HeartbeatMessage(self.device_id)
reply = self.send([pickle.dumps(message)])
if reply:
#.........这里部分代码省略.........
示例13: MDPWorker
# 需要导入模块: from zmq.eventloop.ioloop import PeriodicCallback [as 别名]
# 或者: from zmq.eventloop.ioloop.PeriodicCallback import stop [as 别名]
class MDPWorker(ZMQStream):
def __init__(self, broker, service, io_loop=None):
"""Create and setup an MDP worker.
@param broker A string containing the broker's URL
@param service A string containing the service name
@param io_loop An existing I/O loop object. If None, the default will be used.
"""
self.service=service
self._broker = broker
self.ctx = zmq.Context()
sock = self.ctx.socket(zmq.DEALER)
ZMQStream.__init__(self, sock, io_loop)
# last watchdog timer tick
self.watchdog = 0
# connection callback one-shot
self._conncb = DelayedCallback(self.send_ready, 3000, self.io_loop)
# heartbeat callback..runs continuous when connected
self._hbcb = PeriodicCallback(self.send_heartbeat, 2000, self.io_loop)
# number of connection attempts
self._conn_attempt = 0
# waiting to connect state
self._waiting_to_connect = True
# have we been disconnected? (flags a reconnect attempt)
self.disconnected = False
# connect the socket and send a READY when the io_loop starts
self.connect(self._broker)
self._conncb.start()
def reset_watchdog(self):
"""Private method used to reset the HEARTBEAT watchdog
"""
self.watchdog = time.time()
def disconnect(self):
"""Disconnect from the broker.
"""
logging.info("Disconnected from broker")
self.on_recv(None) # stop message processing
self._conncb.stop() # stop any pending reconnect
self._hbcb.stop() # stop heartbeats
self.disconnected = True
self.io_loop.stop() # stop the I/O loop. If it's used by something else, the caller can restart it
def reconnect(self):
"""Try to reconnect to the broker.
"""
if self.disconnected: # don't try and reconnect, we got an explicit disconnect
return
logging.info("Attempting to reconnect to broker")
self._hbcb.stop()
self._conn_attempt = 0
self._waiting_to_connect = True
try:
self.connect(self._broker)
except ZMQError:
logging.exception()
self.io_loop.stop()
return
self._conncb.start()
def send_ready(self):
"""Send a READY message.
"""
if not self._waiting_to_connect:
# connected already
return
if self.disconnected: # don't try and connect, we got an explicit disconnect
return
logging.debug("Sending READY")
if self._conn_attempt >= 10:
logging.error("10 connection attempts have failed. Giving up.")
return
self._conn_attempt += 1
logging.debug("Connection attempt %i" % self._conn_attempt)
rdy = [b'', MDPW_VER, b'\x01', self.service]
self.on_recv(self.on_message)
self.send_multipart(rdy)
# There is no reply to READY so
# we must assume we are connected unless we see a DISCONNECT
self._waiting_to_connect = False
self._disconed = False
self.reset_watchdog()
self._hbcb.start()
def send_reply(self, client, msg, partial=False):
"""Send a reply to a client. This is typically called from on_request()
@param client The client identifier as passed to on_request()
@param msg The message to send to the client. If this is a list, it's appended to the multipart;
otherwise it is converted to a string and sent as a single frame.
@param partial If this is True, the message is sent as a PARTIAL and at least one
more call must be made to send_reply(). Otherwise a FINAL is sent and
not more calls should be made to send_reply() until another request is processed.
"""
self._hbcb.stop() # stop while sending other messages
if partial:
rply = [b'', MDPW_VER, b'\x03', client, b'']
else:
#.........这里部分代码省略.........
示例14: TaskRouter
# 需要导入模块: from zmq.eventloop.ioloop import PeriodicCallback [as 别名]
# 或者: from zmq.eventloop.ioloop.PeriodicCallback import stop [as 别名]
#.........这里部分代码省略.........
for hr in zombies:
self.log.info("deleting route for %s", hr.host)
del self.route_map[hr.host]
self.stat_inc("dropped_routes")
zombies = []
for rr in self.reply_map.itervalues():
if now - rr.atime > self.reply_timeout:
zombies.append(rr)
for rr in zombies:
self.log.info("deleting reply route for %s", rr.uid)
del self.reply_map[rr.uid]
self.stat_inc("dropped_tasks")
def register_host(self, cmsg):
"""Remember ZMQ route for host"""
route = cmsg.get_route()
msg = cmsg.get_payload(self.xtx)
if not msg:
return
host = msg.host
self.log.info("Got registration for %s", host)
hr = HostRoute(host, route)
self.route_map[hr.host] = hr
self.stat_inc("task.register")
# FIXME: proper reply?
# zans = route + [''] + ['OK']
# self.cclocal.send_multipart(zans)
def send_host(self, cmsg):
"""Send message for task executor on host"""
msg = cmsg.get_payload(self.xtx)
host = msg.task_host
if host not in self.route_map:
self.ccerror(cmsg, "cannot route to %s" % host)
return
inr = cmsg.get_route() # route from/to client
hr = self.route_map[host] # find ZMQ route to host
cmsg.set_route(hr.route) # re-construct message
# send the message
self.log.debug("sending task to %s", host)
cmsg.send_to(self.cclocal)
self.stat_inc("task.send")
# remember ZMQ route for replies
req = cmsg.get_dest()
uid = req.split(".")[2]
rr = ReplyRoute(uid, inr)
self.reply_map[uid] = rr
# send ack to client
rep = TaskReplyMessage(
req="task.reply.%s" % uid, handler=msg["task_handler"], task_id=msg["task_id"], status="forwarded"
)
rcm = self.xtx.create_cmsg(rep)
rcm.set_route(inr)
rcm.send_to(self.cclocal)
self.log.debug("saved client for %r", uid)
def send_reply(self, cmsg):
""" Send reply message back to task requestor """
req = cmsg.get_dest()
uid = req.split(".")[2]
if uid not in self.reply_map:
self.log.info("cannot route back: %s", req)
return
self.log.debug("req: %s", req)
rr = self.reply_map[uid] # find ZMQ route
cmsg.set_route(rr.route) # re-route message
cmsg.send_to(self.cclocal)
rr.atime = time.time() # update feedback time
self.stat_inc("task.reply")
def ccreply(self, rep, creq):
crep = self.xtx.create_cmsg(rep)
crep.take_route(creq)
crep.send_to(self.cclocal)
def ccerror(self, cmsg, errmsg):
self.log.info(errmsg)
rep = ErrorMessage(msg=errmsg)
self.ccreply(rep, cmsg)
def stop(self):
super(TaskRouter, self).stop()
self.log.info("stopping")
self.timer.stop()
示例15: ZmqMaster
# 需要导入模块: from zmq.eventloop.ioloop import PeriodicCallback [as 别名]
# 或者: from zmq.eventloop.ioloop.PeriodicCallback import stop [as 别名]
class ZmqMaster(object, LoggingMixin):
"""
This is the ZMQ Master implementation.
The master will send :class:`DataMessage` object to the workers and receive
the processed messages. Unknown links will then be added to the frontier.
"""
def __init__(self, settings, identity, insocket, outsocket, mgmt, frontier,
log_handler, log_level, io_loop):
"""
Initialize the master.
"""
LoggingMixin.__init__(self, log_handler, log_level)
self._identity = identity
self._io_loop = io_loop or IOLoop.instance()
self._in_stream = ZMQStream(insocket, io_loop)
self._out_stream = ZMQStream(outsocket, io_loop)
self._mgmt = mgmt
self._frontier = frontier
self._running = False
self._available_workers = []
# periodically check if there are pending URIs to crawl
self._periodic_update = PeriodicCallback(self._send_next_uri,
settings.MASTER_PERIODIC_UPDATE_INTERVAL, io_loop=io_loop)
# start this periodic callback when you are waiting for the workers to
# finish
self._periodic_shutdown = PeriodicCallback(self._shutdown_wait, 500,
io_loop=io_loop)
self._shutdown_counter = 0
self._logger.debug("zmqmaster::initialized")
def start(self):
"""
Start the master.
"""
self._mgmt.add_callback(ZMQ_SPYDER_MGMT_WORKER, self._worker_msg)
self._in_stream.on_recv(self._receive_processed_uri)
self._periodic_update.start()
self._running = True
self._logger.debug("zmqmaster::starting...")
def stop(self):
"""
Stop the master gracefully, i.e. stop sending more URIs that should get
processed.
"""
self._logger.debug("zmqmaster::stopping...")
self._running = False
self._periodic_update.stop()
def shutdown(self):
"""
Shutdown the master and notify the workers.
"""
self._logger.debug("zmqmaster::shutdown...")
self.stop()
self._mgmt.publish(topic=ZMQ_SPYDER_MGMT_WORKER,
identity=self._identity, data=ZMQ_SPYDER_MGMT_WORKER_QUIT)
self._frontier.close()
self._periodic_shutdown.start()
def _shutdown_wait(self):
"""
Callback called from `self._periodic_shutdown` in order to wait for the
workers to finish.
"""
self._shutdown_counter += 1
if 0 == len(self._available_workers) or self._shutdown_counter > 5:
self._periodic_shutdown.stop()
self._logger.debug("zmqmaster::bye bye...")
self._io_loop.stop()
def close(self):
"""
Close all open sockets.
"""
self._in_stream.close()
self._out_stream.close()
def finished(self):
"""
Return true if all uris have been processed and the master is ready to
be shut down.
"""
return not self._running
def _worker_msg(self, msg):
"""
Called when a worker has sent a :class:`MgmtMessage`.
"""
if ZMQ_SPYDER_MGMT_WORKER_AVAIL == msg.data:
self._available_workers.append(msg.identity)
self._logger.info("zmqmaster::A new worker is available (%s)" %
msg.identity)
self._send_next_uri()
#.........这里部分代码省略.........