Python PeriodicCallback.stop方法代码示例

# 需要导入模块: from zmq.eventloop.ioloop import PeriodicCallback [as 别名]
# 或者: from zmq.eventloop.ioloop.PeriodicCallback import stop [as 别名]
class WorkerRep(object):

    """Helper class to represent a worker in the broker.

    Instances of this class are used to track the state of the attached worker
    and carry the timers for incomming and outgoing heartbeats.

    :type wid:       str
    :param wid:      the worker id.
    :param service:  service this worker serves
    :type service:   str
    :param stream:   the ZMQStream used to send messages
    :type stream:    ZMQStream
    """

    def __init__(self, wid, service, stream):
        self.id = wid
        self.service = service
        self.multicasts = []
        self.curr_liveness = HB_LIVENESS
        self.stream = stream
        self.last_hb = 0
        self.hb_out_timer = PeriodicCallback(self.send_hb, HB_INTERVAL)
        self.hb_out_timer.start()
        return

    def send_hb(self):
        """Called on every HB_INTERVAL.

        Decrements the current liveness by one.

        Sends heartbeat to worker.
        """
        self.curr_liveness -= 1
        msg = [ self.id, b'', MDP_WORKER_VERSION, b'\x05' ]
        self.stream.send_multipart(msg)
        return

    def on_heartbeat(self):
        """Called when a heartbeat message from the worker was received.

        Sets current liveness to HB_LIVENESS.
        """
        self.curr_liveness = HB_LIVENESS
        return

    def is_alive(self):
        """Returns True when the worker is considered alive.
        """
        return self.curr_liveness > 0

    def shutdown(self):
        """Cleanup worker.

        Stops timer.
        """
        self.hb_out_timer.stop()
        self.hb_out_timer = None
        self.stream = None
        return

# 需要导入模块: from zmq.eventloop.ioloop import PeriodicCallback [as 别名]
# 或者: from zmq.eventloop.ioloop.PeriodicCallback import stop [as 别名]
class DeviceRep(object):
    """
    Helper class to represent a device to a worker
    """

    def __init__(self, device_id, state='unknown'):
        self.id = device_id
        self.state = state
        self.curr_liveness = CLIENT_HB_LIVENESS
        self.hb_timer = PeriodicCallback(self.heartbeat, CLIENT_HB_INTERVAL)
        self.hb_timer.start()
        return

    def heartbeat(self):
        if self.curr_liveness > 0:
            self.curr_liveness -= 1
        if self.curr_liveness == 0:
            self.state = 'dead'
        return

    def on_message_received(self):
        self.curr_liveness = CLIENT_HB_LIVENESS
        return

    def is_alive(self):
        return self.curr_liveness > 0

    def get_state(self):
        return self.state

    def shutdown(self):
        self.hb_timer.stop()
        self.hb_timer = None

# 需要导入模块: from zmq.eventloop.ioloop import PeriodicCallback [as 别名]
# 或者: from zmq.eventloop.ioloop.PeriodicCallback import stop [as 别名]
class TaskState (object):
    """ Tracks task state (with help of watchdog) """

    log = skytools.getLogger ('d:TaskState')

    def __init__ (self, uid, name, info, ioloop, cc, xtx):
        self.uid = uid
        self.name = name
        self.info = info
        self.pidfile = info['config']['pidfile']
        self.ioloop = ioloop
        self.cc = cc
        self.xtx = xtx
        self.timer = None
        self.timer_tick = 1
        self.heartbeat = False
        self.start_time = None
        self.dead_since = None

    def start (self):
        self.start_time = time.time()
        self.timer = PeriodicCallback (self.watchdog, self.timer_tick * 1000, self.ioloop)
        self.timer.start()

    def stop (self):
        try:
            self.log.info ('Signalling %s', self.name)
            skytools.signal_pidfile (self.pidfile, signal.SIGINT)
        except:
            self.log.exception ('signal_pidfile failed: %s', self.pidfile)

    def watchdog (self):
        live = skytools.signal_pidfile (self.pidfile, 0)
        if live:
            self.log.debug ('%s is alive', self.name)
            if self.heartbeat:
                self.send_reply ('running')
        else:
            self.log.info ('%s is over', self.name)
            self.dead_since = time.time()
            self.timer.stop()
            self.timer = None
            self.send_reply ('stopped')

    def ccpublish (self, msg):
        assert isinstance (msg, TaskReplyMessage)
        cmsg = self.xtx.create_cmsg (msg)
        cmsg.send_to (self.cc)

    def send_reply (self, status, feedback = {}):
        msg = TaskReplyMessage(
                req = 'task.reply.%s' % self.uid,
                handler = self.info['task']['task_handler'],
                task_id = self.info['task']['task_id'],
                status = status,
                feedback = feedback)
        self.ccpublish (msg)

# 需要导入模块: from zmq.eventloop.ioloop import PeriodicCallback [as 别名]
# 或者: from zmq.eventloop.ioloop.PeriodicCallback import stop [as 别名]
class DemoApp(ZMQProcess):
    def __init__ (self):
        super(DemoApp, self).__init__()

    def setup(self):
        super(DemoApp, self).setup()

        self.pub, self.pub_addr = self.stream(zmq.PUB, 'tcp://127.0.0.1:%(port)s', True)

        self.sub, sub_addr = self.stream(zmq.SUB, self.pub_addr, False,
                callback=DemoHandler())
        self.heartbeat = PeriodicCallback(self.ping, 1000, self.loop)

    def ping(self):
        print 'SEND PING'
        self.pub.send_multipart(['ping', json.dumps(['ping', time.time()])])

    def local_run(self):
        print 'START HEARTBEAT'
        self.heartbeat.start()

    def stop(self):
        self.heartbeat.stop()
        self.loop.stop()

# 需要导入模块: from zmq.eventloop.ioloop import PeriodicCallback [as 别名]
# 或者: from zmq.eventloop.ioloop.PeriodicCallback import stop [as 别名]

#.........这里部分代码省略.........
        # initialise sockets for communication with workers
        self.dealer_stream, self.dealer_url = self.init_comm (zmq.XREQ, 'inproc://workers-dealer', self.dealer_on_recv)
        self.router_stream, self.router_url = self.init_comm (zmq.XREP, 'inproc://workers-router', self.router_on_recv)

        self.launch_workers()

        self.timer_maint = PeriodicCallback (self.do_maint, self.wparams['maint_period'] * 1000, self.ioloop)
        self.timer_maint.start()

    def init_comm (self, stype, url, cb):
        """ Create socket, stream, etc for communication with workers. """
        sock = self.zctx.socket (stype)
        port = sock.bind_to_random_port (url)
        curl = "%s:%d" % (url, port)
        stream = CCStream (sock, self.ioloop)
        stream.on_recv (cb)
        return (stream, curl)

    def launch_workers (self):
        """ Create and start worker threads. """
        nw = self.cf.getint ('worker-threads', 10)
        for i in range (nw):
            wname = "%s.worker-%i" % (self.hname, i)
            self.log.info ("starting %s", wname)
            w = TailWriter_Worker(
                    wname, self.xtx, self.zctx, self.ioloop,
                    self.dealer_url, self.router_url, self.wparams)
            w.stat_inc = self.stat_inc # XXX
            self.workers.append (w)
            w.start()

    def handle_msg (self, cmsg):
        """ Got message from client, process it. """

        data = cmsg.get_payload (self.xtx)
        if not data: return

        host = data['hostname']
        fn = data['filename']
        st_dev = data.get('st_dev')
        st_ino = data.get('st_ino')

        fi = (host, st_dev, st_ino, fn)
        if fi in self.files:
            fd = self.files[fi]
            if fd.waddr: # already accepted ?
                self.log.trace ("passing %r to %s", fn, fd.wname)
                fd.queue.append (cmsg)
                fd.send_to (self.router_stream)
            else:
                self.log.trace ("queueing %r", fn)
                fd.queue.append (cmsg)
        else:
            fd = FileState (fi, 1)
            self.files[fi] = fd
            self.log.trace ("offering %r", fn)
            self.dealer_stream.send_cmsg (cmsg)

    def dealer_on_recv (self, zmsg):
        """ Got reply from worker via "dealer" connection """
        self.log.warning ("reply via dealer: %s", zmsg)

    def router_on_recv (self, zmsg):
        """ Got reply from worker via "router" connection """
        cmsg = CCMessage (zmsg)
        data = cmsg.get_payload (self.xtx)
        fi = (data['d_hostname'], data['d_st_dev'], data['d_st_ino'], data['d_filename'])
        fd = self.files[fi]
        if fd.waddr is None:
            fd.waddr = zmsg[0]
            fd.wname = data['worker']
        else:
            assert fd.waddr == zmsg[0] and fd.wname == data['worker']
        fd.atime = time.time()
        fd.count -= 1
        assert fd.count >= 0

    def do_maint (self):
        """ Check & flush queues; drop inactive files. """
        self.log.trace ('cleanup')
        now = time.time()
        zombies = []
        for k, fd in self.files.iteritems():
            if fd.queue and fd.waddr:
                self.log.trace ("passing %r to %s", fd.ident, fd.wname)
                fd.send_to (self.router_stream)
            if (fd.count == 0) and (now - fd.atime > 2 * CLOSE_DELAY): # you'd better use msg for this
                self.log.debug ("forgetting %r", fd.ident)
                zombies.append(k)
        for k in zombies:
                self.files.pop(k)

    def stop (self):
        """ Signal workers to shut down. """
        super(TailWriter, self).stop()
        self.log.info ('stopping')
        self.timer_maint.stop()
        for w in self.workers:
            self.log.info ("signalling %s", w.name)
            w.stop()

# 需要导入模块: from zmq.eventloop.ioloop import PeriodicCallback [as 别名]
# 或者: from zmq.eventloop.ioloop.PeriodicCallback import stop [as 别名]
class MDPWorker(object):

    """Class for the MDP worker side.

    Thin encapsulation of a zmq.XREQ socket.
    Provides a send method with optional timeout parameter.

    Will use a timeout to indicate a broker failure.
    """

    _proto_version = b'MDPW01'

    # TODO: integrate that into API
    HB_INTERVAL = 1000  # in milliseconds
    HB_LIVENESS = 3    # HBs to miss before connection counts as dead

    def __init__(self, context, endpoint, service):
        """Initialize the MDPWorker.

        context is the zmq context to create the socket from.
        service is a byte-string with the service name.
        """
        self.context = context
        self.endpoint = endpoint
        self.service = service
        self.stream = None
        self._tmo = None
        self.need_handshake = True
        self.ticker = None
        self._delayed_cb = None
        self._create_stream()
        return

    def _create_stream(self):
        """Helper to create the socket and the stream.
        """
        socket = self.context.socket(zmq.XREQ)
        ioloop = IOLoop.instance()
        self.stream = ZMQStream(socket, ioloop)
        self.stream.on_recv(self._on_message)
        self.stream.socket.setsockopt(zmq.LINGER, 0)
        self.stream.connect(self.endpoint)
        self.ticker = PeriodicCallback(self._tick, self.HB_INTERVAL)
        self._send_ready()
        self.ticker.start()
        return

    def _send_ready(self):
        """Helper method to prepare and send the workers READY message.
        """
        ready_msg = [ b'', self._proto_version, chr(1), self.service ]
        self.stream.send_multipart(ready_msg)
        self.curr_liveness = self.HB_LIVENESS
        return

    def _tick(self):
        """Method called every HB_INTERVAL milliseconds.
        """
        self.curr_liveness -= 1
##         print '%.3f tick - %d' % (time.time(), self.curr_liveness)
        self.send_hb()
        if self.curr_liveness >= 0:
            return
        print '%.3f lost connection' % time.time()
        # ouch, connection seems to be dead
        self.shutdown()
        # try to recreate it
        self._delayed_cb = DelayedCallback(self._create_stream, 5000)
        self._delayed_cb.start()
        return

    def send_hb(self):
        """Construct and send HB message to broker.
        """
        msg = [ b'', self._proto_version, chr(4) ]
        self.stream.send_multipart(msg)
        return

    def shutdown(self):
        """Method to deactivate the worker connection completely.

        Will delete the stream and the underlying socket.
        """
        if self.ticker:
            self.ticker.stop()
            self.ticker = None
        if not self.stream:
            return
        self.stream.socket.close()
        self.stream.close()
        self.stream = None
        self.timed_out = False
        self.need_handshake = True
        self.connected = False
        return

    def reply(self, msg):
        """Send the given message.

        msg can either be a byte-string or a list of byte-strings.
#.........这里部分代码省略.........

# 需要导入模块: from zmq.eventloop.ioloop import PeriodicCallback [as 别名]
# 或者: from zmq.eventloop.ioloop.PeriodicCallback import stop [as 别名]
class Worker(object):

    """Class for the MDP worker side.

    Thin encapsulation of a zmq.DEALER socket.
    Provides a send method with optional timeout parameter.

    Will use a timeout to indicate a broker failure.
    """
    max_forks = 10

    ipc = 'ipc:///tmp/zmq-rpc-'+str(uuid4())
    HB_INTERVAL = 1000  # in milliseconds
    HB_LIVENESS = 3    # HBs to miss before connection counts as dead

    def __init__(self, context, endpoint, service, multicasts=()):
        """Initialize the MDPWorker.

        :param context:    is the zmq context to create the socket from
        :type context:     zmq.Context
        :param service:    service name - you can put hostname here
        :type service:     str
        :param multicasts: list of groups to subscribe
        :type multicasts:  list
        """
        self.context = context
        self.endpoint = endpoint
        self.service = service.encode('utf-8')  # convert to byte-string - required in python 3
        self.multicasts = [m.encode('utf-8') for m in multicasts]  # convert to byte-string
        self.stream = None
        self._tmo = None
        self.need_handshake = True
        self.ticker = None
        self._delayed_cb = None
        self._create_stream()
        self.forks = []
        self.curr_liveness = self.HB_LIVENESS

        socket = self.context.socket(zmq.ROUTER)
        socket.bind(self.ipc)
        self.stream_w = ZMQStream(socket)
        self.stream_w.on_recv(self._on_fork_response)
        self.reply_socket = None
        return

    def _create_stream(self):
        """Helper to create the socket and the stream.
        """
        self.on_log_event('broker.connect', 'Trying to connect do broker')
        socket = self.context.socket(zmq.DEALER)
        ioloop = IOLoop.instance()
        self.stream = ZMQStream(socket, ioloop)
        self.stream.on_recv(self._on_message)
        self.stream.socket.setsockopt(zmq.LINGER, 0)
        self.stream.connect(self.endpoint)
        self.ticker = PeriodicCallback(self._tick, self.HB_INTERVAL)
        self._send_ready()
        for m in self.multicasts:
            self._register_worker_to_multicast(m)
        self.ticker.start()
        return

    def _tick(self):
        """Method called every HB_INTERVAL milliseconds.
        """
        self.curr_liveness -= 1
        self.send_hb()
        if self.curr_liveness >= 0:
            return
        # ouch, connection seems to be dead
        self.on_log_event('broker.timeout', 'Connection to broker timeouted, disconnecting')
        self.shutdown(False)
        # try to recreate it
        self._delayed_cb = DelayedCallback(self._create_stream, 5000)
        self._delayed_cb.start()
        return

    def send_hb(self):
        """Construct and send HB message to broker.
        """
        msg = [b'', MDP_WORKER_VERSION, b'\x05']
        self.stream.send_multipart(msg)
        return

    def shutdown(self, final=True):
        """Method to deactivate the worker connection completely.

        Will delete the stream and the underlying socket.

        :param final: if shutdown is final and we want to close all sockets
        :type final:  bool
        """

        if self.ticker:
            self.ticker.stop()
            self.ticker = None
        if not self.stream:
            return

        self.stream.on_recv(None)
#.........这里部分代码省略.........

# 需要导入模块: from zmq.eventloop.ioloop import PeriodicCallback [as 别名]
# 或者: from zmq.eventloop.ioloop.PeriodicCallback import stop [as 别名]
class MQRep(object):

    """Class for the MDP worker side.

    Thin encapsulation of a zmq.DEALER socket.
    Provides a send method with optional timeout parameter.

    Will use a timeout to indicate a broker failure.
    """

    _proto_version = b'MDPW01'

    # TODO: integrate that into API
    HB_INTERVAL = 1000  # in milliseconds
    HB_LIVENESS = 3    # HBs to miss before connection counts as dead

    def __init__(self, context, service):
        """Initialize the MDPWorker.

        context is the zmq context to create the socket from.
        service is a byte-string with the service name.
        """
        if DEBUG:
            print("MQRep > __init__")
        cfg = Loader('mq').load()
        config = dict(cfg[1])
        if config['ip'].strip() == "*":
            config['ip'] = get_ip()
        self.endpoint = "tcp://{0}:{1}".format(config['ip'], config['req_rep_port'])
        self.context = context
        self.service = service
        self.stream = None
        self._tmo = None
        self.need_handshake = True
        self.ticker = None
        self._delayed_cb = None
        self._create_stream()

        ### patch fritz
        self._reconnect_in_progress = False
        ### end patch fritz
        return

    def _create_stream(self):
        """Helper to create the socket and the stream.
        """
        if DEBUG:
            print("MQRep > _create_stream")
        socket = ZmqSocket(self.context, zmq.DEALER)
        ioloop = IOLoop.instance()
        self.stream = ZMQStream(socket, ioloop)
        self.stream.on_recv(self._on_mpd_message)
        self.stream.socket.setsockopt(zmq.LINGER, 0)
        self.stream.connect(self.endpoint)
        if self.ticker != None:
            if DEBUG:
                print("MQRep > _create_stream - stop ticker")
            self.ticker.stop()
        self.ticker = PeriodicCallback(self._tick, self.HB_INTERVAL)
        self._send_ready()
        self.ticker.start()
        return

    def _send_ready(self):
        """Helper method to prepare and send the workers READY message.
        """
        if DEBUG:
            print("MQREP > _send_ready")
        ready_msg = [ b'', self._proto_version, b'\x01', self.service ]
        self.stream.send_multipart(ready_msg)
        self.curr_liveness = self.HB_LIVENESS
        if DEBUG:
            print("MQREP > _send_ready > curr_liveness <= {0}".format(self.HB_LIVENESS))
        return

    def _tick(self):
        """Method called every HB_INTERVAL milliseconds.
        """
        if DEBUG:
            print("MQREP > _tick")
        self.curr_liveness -= 1
        if DEBUG:
            print('MQREP > _tick - {0} tick = {1}'.format(time.time(), self.curr_liveness))
        self.send_hb()
        if self.curr_liveness >= 0:
            return
        if DEBUG:
            print('MQREP > _tick - {0} lost connection'.format(time.time()))
        # ouch, connection seems to be dead
        self.shutdown()
        # try to recreate it
        self._delayed_cb = DelayedCallback(self._create_stream, self.HB_INTERVAL)
        self._delayed_cb.start()
        return

    def send_hb(self):
        """Construct and send HB message to broker.
        """
        msg = [ b'', self._proto_version, b'\x04' ]
        self.stream.send_multipart(msg)
#.........这里部分代码省略.........

# 需要导入模块: from zmq.eventloop.ioloop import PeriodicCallback [as 别名]
# 或者: from zmq.eventloop.ioloop.PeriodicCallback import stop [as 别名]
class Master(object):

    def __init__(self, frontier,
            data_in_sock='ipc:///tmp/robot-data-w2m.sock',
            data_out_sock='ipc:///tmp/robot-data-m2w.sock',
            msg_in_sock='ipc:///tmp/robot-msg-w2m.sock',
            msg_out_sock='ipc:///tmp/robot-msg-m2w.sock',
            io_loop=None):
        self.identity = 'master:%s:%s' % (socket.gethostname(), os.getpid())

        context = zmq.Context()

        self._io_loop = io_loop or IOLoop.instance()

        self._in_socket = context.socket(zmq.SUB)
        self._in_socket.setsockopt(zmq.SUBSCRIBE, '')
        self._in_socket.bind(data_in_sock)
        self._in_stream = ZMQStream(self._in_socket, io_loop)

        self._out_socket = context.socket(zmq.PUSH)
        self._out_socket.bind(data_out_sock)
        self._out_stream = ZMQStream(self._out_socket, io_loop)

        self._online_workers = set()
        self._running = False

        self._updater = PeriodicCallback(self._send_next, 100, io_loop=io_loop)
        self._reloader = PeriodicCallback(self.reload, 1000, io_loop=io_loop)

        self.frontier = frontier
        self.messenger = ServerMessenger(msg_in_sock, msg_out_sock,
                context, io_loop)

    def start(self):
        logging.info('[%s] starting', self.identity)
        self.messenger.add_callback(CTRL_MSG_WORKER, self._on_worker_msg)
        self.messenger.start()

        self._in_stream.on_recv(self._on_receive_processed)
        self._updater.start()
        self._reloader.start()
        self._running = True

    def stop(self):
        self._running = False
        self._reloader.stop()
        self._updater.stop()
        self.messenger.stop()
#        self.messenger.publish(CTRL_MSG_WORKER, self.identity,
#                CTRL_MSG_WORKER_QUIT)

    def close(self):
        self._in_stream.close()
        self._in_socket.close()
        self._out_stream.close()
        self._out_socket.close()
        self.messenger.close()

    def reload(self):
        pass

    def _on_worker_msg(self, msg):
        if msg.data == CTRL_MSG_WORKER_ONLINE:
            self._online_workers.add(msg.identity)
            logging.info('[%s] append [%s]', self.identity, msg.identity)
            self._send_next()

#        if msg.data == CTRL_MSG_WORKER_QUIT_ACK:
#            if msg.identity in self._online_workers:
#                self._online_workers.remove(msg.identity)

    def _send_next(self):
        if not self._running:
            return

        worker_num = len(self._online_workers)

        if self._running and worker_num > 0:
            while self._out_stream._send_queue.qsize() < worker_num * 4:
                request = self.frontier.get_next_request()
                if not request:
                    break

                msg = RequestMessage(self.identity, request)
                self._out_stream.send_multipart(msg.serialize())
                logging.debug('[%s] send request(%s)',
                        self.identity, request.url)

                self.frontier.reload_request(request)

    def _on_receive_processed(self, zmq_msg):
        msg = ResponseMessage.deserialize(zmq_msg)
        request = msg.response.request
        logging.debug('[%s] receive response(%s)', self.identity, request.url)
        self._send_next()

# 需要导入模块: from zmq.eventloop.ioloop import PeriodicCallback [as 别名]
# 或者: from zmq.eventloop.ioloop.PeriodicCallback import stop [as 别名]
class DeviceServiceManager(MDPWorker):

    _connected_devices = {}

    def __init__(self, context, endpoint, service):
        """Overridden initializer for MDPWorker.
        Adds the device_timer to manage connected devices
        """
        session = Session()
        session.query(Device).update({Device.connected: False})
        session.commit()
        self.device_timer = None
        super(DeviceServiceManager, self).__init__(context, endpoint, service)
        return

    def _create_stream(self):
        """Overidden _create_stream for MDPWorker
        Adds the device_timer to manage connected devices
        """
        self.device_timer = PeriodicCallback(self.device_watcher,
                                             CLIENT_HB_INTERVAL)
        self.device_timer.start()
        super(DeviceServiceManager, self)._create_stream()
        return

    def shutdown(self):
        """Overidden shutdown for MDPWorker
        Adds the device_timer to manage connected devices
        """
        self.device_timer.stop()
        self.device_timer = None
        super(DeviceServiceManager, self).shutdown()
        return

    def device_watcher(self):
        for device in self._connected_devices.values():
            if not device.is_alive():
                session = Session()
                session.query(Device).filter_by(id=device.id) \
                                     .update({Device.connected: False})
                session.commit()
                device.shutdown()
                del self._connected_devices[device.id]
        return

    def on_heartbeat(self, did):
        if did in self._connected_devices:
            if self._connected_devices[did].is_alive():
                self._connected_devices[did].on_message_received()
            return self._connected_devices[did].get_state()
        else:
            session = Session()
            device = session.query(Device).filter_by(id=did).first()

            if not device:
                session.close()
                # signals the device to send a create message
                return 'unknown'
            else:
                self._connected_devices[did] = DeviceRep(did,
                                                         state='connected')
                device.connected = True
                session.commit()
                return 'connected'
        return

    def on_create(self, did, msg):
        imported_device = pickle.loads(msg[0])

        device = Device(imported_device['id'],
                        version=imported_device['version'])

        device.connected = True
        device.user_configured = False
        device.database_service = imported_device['database_service']
        device.device_service = imported_device['device_service']
        device.grainbin_service = imported_device['grainbin_service']
        device.grainbin_count = imported_device['grainbin_count']

        session = Session()
        session.add(device)

        if device.grainbin_service:
            grainbins = []
            for x in range(device.grainbin_count):
                id = device.id + '.' + str(x).zfill(2)
                grainbin = Grainbin(id, device.id, x)
                grainbins.append(grainbin)
            session.add_all(grainbins)

        session.commit()
        session.close()

        return "added"

    def on_remove(self, did, msg):
        if did in self._connected_devices:
            self._connected_devices[did].shutdown()
            del self._connected_devices[did]

#.........这里部分代码省略.........

# 需要导入模块: from zmq.eventloop.ioloop import PeriodicCallback [as 别名]
# 或者: from zmq.eventloop.ioloop.PeriodicCallback import stop [as 别名]

#.........这里部分代码省略.........

        for url in self.cf.getlist ("ping-remotes", ""):
            sock = self._make_socket (url)
            self.stream[url] = CCStream (sock, ccscript.ioloop, qmaxsize = self.zmq_hwm)
            self.stream[url].on_recv (self.on_recv)
            self.echoes[url] = EchoState (url)
            self.log.debug ("will ping %s", url)

        self.timer = PeriodicCallback (self.ping, self.ping_tick * 1000, self.ioloop)
        self.timer.start()

    def _make_socket (self, url):
        """ Create socket for pinging remote CC. """
        sock = self.zctx.socket (zmq.XREQ)
        sock.setsockopt (zmq.HWM, self.zmq_hwm)
        sock.setsockopt (zmq.LINGER, self.zmq_linger)
        sock.connect (url)
        return sock

    def on_recv (self, zmsg):
        """ Got reply from a remote CC, process it. """
        try:
            self.log.trace ("%r", zmsg)
            cmsg = CCMessage (zmsg)
            req = cmsg.get_dest()
            if req == "echo.response":
                self.process_response (cmsg)
            else:
                self.log.warn ("unknown msg: %s", req)
        except:
            self.log.exception ("crashed, dropping msg")

    def handle_msg (self, cmsg):
        """ Got a message, process it. """

        self.log.trace ("%r", cmsg)
        req = cmsg.get_dest()

        if req == "echo.request":
            self.process_request (cmsg)
        else:
            self.log.warn ("unknown msg: %s", req)

    def process_request (self, cmsg):
        """ Ping received, respond with pong. """

        msg = cmsg.get_payload (self.xtx)
        if not msg: return

        rep = EchoResponseMessage(
                orig_hostname = msg['hostname'],
                orig_target = msg['target'],
                orig_time = msg['time'])
        rcm = self.xtx.create_cmsg (rep)
        rcm.take_route (cmsg)
        rcm.send_to (self.cclocal)

    def process_response (self, cmsg):
        """ Pong received, evaluate it. """

        msg = cmsg.get_payload (self.xtx)
        if not msg: return

        url = msg.orig_target
        if url not in self.echoes:
            self.log.warn ("unknown pong: %s", url)
            return
        echo = self.echoes[url]
        echo.update_pong (msg)

        rtt = echo.time_pong - msg.orig_time
        if msg.orig_time == echo.time_ping:
            self.log.trace ("echo time: %f s (%s)", rtt, url)
        elif rtt <= 5 * self.ping_tick:
            self.log.debug ("late pong: %f s (%s)", rtt, url)
        else:
            self.log.info ("too late pong: %f s (%s)", rtt, url)

    def send_request (self, url):
        """ Send ping to remote CC. """
        msg = EchoRequestMessage(
                target = url)
        cmsg = self.xtx.create_cmsg (msg)
        self.stream[url].send_cmsg (cmsg)
        self.echoes[url].update_ping (msg)
        self.log.trace ("%r", msg)

    def ping (self):
        """ Echo requesting and monitoring. """
        self.log.trace ("")
        for url in self.stream:
            echo = self.echoes[url]
            if echo.time_ping - echo.time_pong > 5 * self.ping_tick:
                self.log.warn ("no pong from %s for %f s", url, echo.time_ping - echo.time_pong)
            self.send_request (url)

    def stop (self):
        super(Echo, self).stop()
        self.log.info ("stopping")
        self.timer.stop()

# 需要导入模块: from zmq.eventloop.ioloop import PeriodicCallback [as 别名]
# 或者: from zmq.eventloop.ioloop.PeriodicCallback import stop [as 别名]
class DeviceConnection(object):

    SERVICE_NAME = 'device'  # service to connect to
    TIMEOUT = 5  # time to wait for answer in seconds

    # Number of connections to try before restarting when in incorrect state
    CONNECTION_ATTEMPS = 3
    HB_INTERVAL = 1000 * 10  # in milliseconds

    def __init__(self, context, device_id, address):

        self.context = context
        self.device_id = device_id
        self.address = address

        self.ticker = None
        self.updater = None
        self.socket = None
        self.can_send = False

        self._restart()
        self.heartbeat()

        return

    def _restart(self):

        self.shutdown()

        self.socket = self.context.socket(zmq.REQ)
        self.socket.setsockopt(zmq.LINGER, 0)
        self.socket.connect(self.address)

        self.can_send = True
        self.connection_attempts = 0
        self.device_state = 'unknown'
        self.ticker = PeriodicCallback(self.heartbeat,
                                       DeviceConnection.HB_INTERVAL)
        self.ticker.start()

        # delay = seconds_till_next('hour', duration=1) + 2  # 2 second buffer
        delay = 60  # send update one minute after startup
        self.updater = DelayedCallback(self.update, delay * 1000)
        self.updater.start()
        return

    def shutdown(self):

        if self.ticker:
            self.ticker.stop()
            self.ticker = None

        if self.updater:
            self.updater.stop()

        if self.socket:
            self.socket.close()
            self.socket = None

        self.can_send = False
        return

    def send(self, message):

        if not self.can_send:
            self.connection_attempts += 1
            logger.error("DeviceConnection is not in state to send")
            return None
        else:
            self.connection_attempts = 0

        self.can_send = False

        logger.debug("DeviceConnection sending {0}".format(message))

        reply = mdp_request(self.socket,
                            DeviceConnection.SERVICE_NAME,
                            message,
                            DeviceConnection.TIMEOUT)

        if reply:
            logger.debug("DeviceConnection reply received: {0}".format(reply))
            self.can_send = True
            return reply
        else:
            # Timeout! Will be restarted at next heartbeat
            logger.warn("DeviceConnection timeout. Will be restarted at next heartbeat")
            self.connection_attempts = DeviceConnection.CONNECTION_ATTEMPS
            return None

    def heartbeat(self):

        if self.connection_attempts >= DeviceConnection.CONNECTION_ATTEMPS:
            logger.warn("DeviceConnection attempts max reached. Restarting connection.")
            self._restart()

        message = HeartbeatMessage(self.device_id)
        reply = self.send([pickle.dumps(message)])

        if reply:
#.........这里部分代码省略.........

# 需要导入模块: from zmq.eventloop.ioloop import PeriodicCallback [as 别名]
# 或者: from zmq.eventloop.ioloop.PeriodicCallback import stop [as 别名]
class MDPWorker(ZMQStream):
    def __init__(self, broker, service, io_loop=None):
        """Create and setup an MDP worker.
           @param broker A string containing the broker's URL
           @param service A string containing the service name
           @param io_loop An existing I/O loop object. If None, the default will be used.
        """
        self.service=service
        self._broker = broker

        self.ctx = zmq.Context()
        sock = self.ctx.socket(zmq.DEALER)
        ZMQStream.__init__(self, sock, io_loop)
        # last watchdog timer tick
        self.watchdog = 0
        # connection callback one-shot
        self._conncb = DelayedCallback(self.send_ready, 3000, self.io_loop)
        # heartbeat callback..runs continuous when connected
        self._hbcb = PeriodicCallback(self.send_heartbeat, 2000, self.io_loop)
        # number of connection attempts
        self._conn_attempt = 0
        # waiting to connect state
        self._waiting_to_connect = True
        # have we been disconnected? (flags a reconnect attempt)
        self.disconnected = False

        # connect the socket and send a READY when the io_loop starts
        self.connect(self._broker)
        self._conncb.start()

    def reset_watchdog(self):
        """Private method used to reset the HEARTBEAT watchdog
        """
        self.watchdog = time.time()

    def disconnect(self):
        """Disconnect from the broker.
        """
        logging.info("Disconnected from broker")
        self.on_recv(None) # stop message processing
        self._conncb.stop() # stop any pending reconnect
        self._hbcb.stop() # stop heartbeats
        self.disconnected = True
        self.io_loop.stop() # stop the I/O loop. If it's used by something else, the caller can restart it

    def reconnect(self):
        """Try to reconnect to the broker.
        """
        if self.disconnected: # don't try and reconnect, we got an explicit disconnect
            return
        logging.info("Attempting to reconnect to broker")
        self._hbcb.stop()
        self._conn_attempt = 0
        self._waiting_to_connect = True
        try:
            self.connect(self._broker)
        except ZMQError:
            logging.exception()
            self.io_loop.stop()
            return
        self._conncb.start()

    def send_ready(self):
        """Send a READY message.
        """
        if not self._waiting_to_connect:
            # connected already
            return
        if self.disconnected: # don't try and connect, we got an explicit disconnect
            return
        logging.debug("Sending READY")
        if self._conn_attempt >= 10:
            logging.error("10 connection attempts have failed. Giving up.")
            return
        self._conn_attempt += 1
        logging.debug("Connection attempt %i" % self._conn_attempt)
        rdy = [b'', MDPW_VER, b'\x01', self.service]
        self.on_recv(self.on_message)
        self.send_multipart(rdy)
        # There is no reply to READY so
        # we must assume we are connected unless we see a DISCONNECT
        self._waiting_to_connect = False
        self._disconed = False
        self.reset_watchdog()
        self._hbcb.start()

    def send_reply(self, client, msg, partial=False):
        """Send a reply to a client. This is typically called from on_request()
           @param client The client identifier as passed to on_request()
           @param msg The message to send to the client. If this is a list, it's appended to the multipart;
                      otherwise it is converted to a string and sent as a single frame.
           @param partial If this is True, the message is sent as a PARTIAL and at least one
                          more call must be made to send_reply(). Otherwise a FINAL is sent and
                          not more calls should be made to send_reply() until another request is processed.
                   
        """
        self._hbcb.stop() # stop while sending other messages
        if partial:
            rply = [b'', MDPW_VER, b'\x03', client, b'']
        else:
#.........这里部分代码省略.........

# 需要导入模块: from zmq.eventloop.ioloop import PeriodicCallback [as 别名]
# 或者: from zmq.eventloop.ioloop.PeriodicCallback import stop [as 别名]

#.........这里部分代码省略.........
        for hr in zombies:
            self.log.info("deleting route for %s", hr.host)
            del self.route_map[hr.host]
            self.stat_inc("dropped_routes")

        zombies = []
        for rr in self.reply_map.itervalues():
            if now - rr.atime > self.reply_timeout:
                zombies.append(rr)
        for rr in zombies:
            self.log.info("deleting reply route for %s", rr.uid)
            del self.reply_map[rr.uid]
            self.stat_inc("dropped_tasks")

    def register_host(self, cmsg):
        """Remember ZMQ route for host"""

        route = cmsg.get_route()
        msg = cmsg.get_payload(self.xtx)
        if not msg:
            return
        host = msg.host
        self.log.info("Got registration for %s", host)
        hr = HostRoute(host, route)
        self.route_map[hr.host] = hr

        self.stat_inc("task.register")

        # FIXME: proper reply?
        # zans = route + [''] + ['OK']
        # self.cclocal.send_multipart(zans)

    def send_host(self, cmsg):
        """Send message for task executor on host"""

        msg = cmsg.get_payload(self.xtx)
        host = msg.task_host

        if host not in self.route_map:
            self.ccerror(cmsg, "cannot route to %s" % host)
            return

        inr = cmsg.get_route()  # route from/to client
        hr = self.route_map[host]  # find ZMQ route to host
        cmsg.set_route(hr.route)  # re-construct message

        # send the message
        self.log.debug("sending task to %s", host)
        cmsg.send_to(self.cclocal)
        self.stat_inc("task.send")

        # remember ZMQ route for replies
        req = cmsg.get_dest()
        uid = req.split(".")[2]
        rr = ReplyRoute(uid, inr)
        self.reply_map[uid] = rr

        # send ack to client
        rep = TaskReplyMessage(
            req="task.reply.%s" % uid, handler=msg["task_handler"], task_id=msg["task_id"], status="forwarded"
        )
        rcm = self.xtx.create_cmsg(rep)
        rcm.set_route(inr)
        rcm.send_to(self.cclocal)

        self.log.debug("saved client for %r", uid)

    def send_reply(self, cmsg):
        """ Send reply message back to task requestor """

        req = cmsg.get_dest()
        uid = req.split(".")[2]

        if uid not in self.reply_map:
            self.log.info("cannot route back: %s", req)
            return

        self.log.debug("req: %s", req)

        rr = self.reply_map[uid]  # find ZMQ route
        cmsg.set_route(rr.route)  # re-route message
        cmsg.send_to(self.cclocal)
        rr.atime = time.time()  # update feedback time

        self.stat_inc("task.reply")

    def ccreply(self, rep, creq):
        crep = self.xtx.create_cmsg(rep)
        crep.take_route(creq)
        crep.send_to(self.cclocal)

    def ccerror(self, cmsg, errmsg):
        self.log.info(errmsg)
        rep = ErrorMessage(msg=errmsg)
        self.ccreply(rep, cmsg)

    def stop(self):
        super(TaskRouter, self).stop()
        self.log.info("stopping")
        self.timer.stop()

# 需要导入模块: from zmq.eventloop.ioloop import PeriodicCallback [as 别名]
# 或者: from zmq.eventloop.ioloop.PeriodicCallback import stop [as 别名]
class ZmqMaster(object, LoggingMixin):
    """
    This is the ZMQ Master implementation.

    The master will send :class:`DataMessage` object to the workers and receive
    the processed messages. Unknown links will then be added to the frontier.
    """

    def __init__(self, settings, identity, insocket, outsocket, mgmt, frontier,
            log_handler, log_level, io_loop):
        """
        Initialize the master.
        """
        LoggingMixin.__init__(self, log_handler, log_level)
        self._identity = identity
        self._io_loop = io_loop or IOLoop.instance()

        self._in_stream = ZMQStream(insocket, io_loop)
        self._out_stream = ZMQStream(outsocket, io_loop)

        self._mgmt = mgmt
        self._frontier = frontier

        self._running = False
        self._available_workers = []

        # periodically check if there are pending URIs to crawl
        self._periodic_update = PeriodicCallback(self._send_next_uri,
                settings.MASTER_PERIODIC_UPDATE_INTERVAL, io_loop=io_loop)
        # start this periodic callback when you are waiting for the workers to
        # finish
        self._periodic_shutdown = PeriodicCallback(self._shutdown_wait, 500,
                io_loop=io_loop)
        self._shutdown_counter = 0
        self._logger.debug("zmqmaster::initialized")

    def start(self):
        """
        Start the master.
        """
        self._mgmt.add_callback(ZMQ_SPYDER_MGMT_WORKER, self._worker_msg)
        self._in_stream.on_recv(self._receive_processed_uri)
        self._periodic_update.start()
        self._running = True
        self._logger.debug("zmqmaster::starting...")

    def stop(self):
        """
        Stop the master gracefully, i.e. stop sending more URIs that should get
        processed.
        """
        self._logger.debug("zmqmaster::stopping...")
        self._running = False
        self._periodic_update.stop()

    def shutdown(self):
        """
        Shutdown the master and notify the workers.
        """
        self._logger.debug("zmqmaster::shutdown...")
        self.stop()
        self._mgmt.publish(topic=ZMQ_SPYDER_MGMT_WORKER,
                identity=self._identity, data=ZMQ_SPYDER_MGMT_WORKER_QUIT)
        self._frontier.close()
        self._periodic_shutdown.start()

    def _shutdown_wait(self):
        """
        Callback called from `self._periodic_shutdown` in order to wait for the
        workers to finish.
        """
        self._shutdown_counter += 1
        if 0 == len(self._available_workers) or self._shutdown_counter > 5:
            self._periodic_shutdown.stop()
            self._logger.debug("zmqmaster::bye bye...")
            self._io_loop.stop()

    def close(self):
        """
        Close all open sockets.
        """
        self._in_stream.close()
        self._out_stream.close()

    def finished(self):
        """
        Return true if all uris have been processed and the master is ready to
        be shut down.
        """
        return not self._running

    def _worker_msg(self, msg):
        """
        Called when a worker has sent a :class:`MgmtMessage`.
        """
        if ZMQ_SPYDER_MGMT_WORKER_AVAIL == msg.data:
            self._available_workers.append(msg.identity)
            self._logger.info("zmqmaster::A new worker is available (%s)" %
                    msg.identity)
            self._send_next_uri()
#.........这里部分代码省略.........