Python ZMQStream.connect方法代码示例

# 需要导入模块: from zmq.eventloop.zmqstream import ZMQStream [as 别名]
# 或者: from zmq.eventloop.zmqstream.ZMQStream import connect [as 别名]
class Subscriber(object):

    def __init__(self, context, sub_address, sub_topics):
        self.context = context
        self.subscriber_address = sub_address
        self.subscriber_topics = sub_topics

        socket = self.context.socket(zmq.SUB)
        ioloop = IOLoop.instance()
        self.subscriber = ZMQStream(socket, ioloop)
        self.subscriber.setsockopt(zmq.LINGER, 0)
        self.subscriber.on_recv(callback=self.subscriber_recv)
        self.subscriber.setsockopt(zmq.SUBSCRIBE, "")
        self.subscriber.connect(self.subscriber_address)

        return

    def shutdown(self):

        self.subscriber.on_recv(callback=None)
        self.subscriber.socket.disconnect(self.subscriber_address)
        self.subscriber = None

        return

    def subscriber_recv(self, msg):

        return

# 需要导入模块: from zmq.eventloop.zmqstream import ZMQStream [as 别名]
# 或者: from zmq.eventloop.zmqstream.ZMQStream import connect [as 别名]
class Worker(object):

    def __init__(endpoints):
        context = zmq.Context()
        socket = context.socket(zmq.REQ)
        ioloop = IOLoop.instance()
        self.endpoints = endpoints
        self.stream = ZMQStream(socket, ioloop)
        self.stream.on_recv(self._on_message)
        self.stream.socket.setsocketopt(zmq.LINGER, 0)
        for endpoint in endpoints:
            self.stream.connect(endpoint)
        self._send_ready()

# 需要导入模块: from zmq.eventloop.zmqstream import ZMQStream [as 别名]
# 或者: from zmq.eventloop.zmqstream.ZMQStream import connect [as 别名]
def launch_scheduler(in_addr, out_addr, mon_addr, not_addr, config=None,logname='ZMQ', 
                            log_addr=None, loglevel=logging.DEBUG, scheme='lru',
                            identity=b'task'):
    from zmq.eventloop import ioloop
    from zmq.eventloop.zmqstream import ZMQStream
    
    if config:
        # unwrap dict back into Config
        config = Config(config)

    ctx = zmq.Context()
    loop = ioloop.IOLoop()
    ins = ZMQStream(ctx.socket(zmq.XREP),loop)
    ins.setsockopt(zmq.IDENTITY, identity)
    ins.bind(in_addr)
    
    outs = ZMQStream(ctx.socket(zmq.XREP),loop)
    outs.setsockopt(zmq.IDENTITY, identity)
    outs.bind(out_addr)
    mons = ZMQStream(ctx.socket(zmq.PUB),loop)
    mons.connect(mon_addr)
    nots = ZMQStream(ctx.socket(zmq.SUB),loop)
    nots.setsockopt(zmq.SUBSCRIBE, '')
    nots.connect(not_addr)
    
    scheme = globals().get(scheme, None)
    # setup logging
    if log_addr:
        connect_logger(logname, ctx, log_addr, root="scheduler", loglevel=loglevel)
    else:
        local_logger(logname, loglevel)
    
    scheduler = TaskScheduler(client_stream=ins, engine_stream=outs,
                            mon_stream=mons, notifier_stream=nots,
                            scheme=scheme, loop=loop, logname=logname,
                            config=config)
    scheduler.start()
    try:
        loop.start()
    except KeyboardInterrupt:
        print ("interrupted, exiting...", file=sys.__stderr__)

# 需要导入模块: from zmq.eventloop.zmqstream import ZMQStream [as 别名]
# 或者: from zmq.eventloop.zmqstream.ZMQStream import connect [as 别名]
class LocalRequestProxy:
    '''
    This class is responsible for routing client requests coming from a
    particular server to the RouterPubSubProxy, which will route them to the
    workers.
    '''
    def __init__(self, front_end_name, back_end_name, loop):
        '''
        Initializes an instance of LocalRequestProxy

        @param front_end_name - name of the front end socket. It will be
                                initialized with the Router socket.
        @param back_end_name - name of the back end socket. It will be
                               initialized with the Dealer socket.
        @param loop - zmq IOLoop
        '''
        self._loop = loop

        ctx = zmq.Context.instance()

        # Create the front end stream
        front_address = ZmqAddress(chan_name=front_end_name, transport=INPROC)
        self._front_end = ZMQStream(ctx.socket(zmq.ROUTER), io_loop=loop)
        self._front_end.setsockopt(zmq.ROUTER_MANDATORY, 1)
        self._front_end.bind(front_address.zmq_url())

        # Create the back end stream
        back_address = ZmqAddress(chan_name=back_end_name)
        self._back_end = ZMQStream(ctx.socket(zmq.DEALER), io_loop=loop)
        self._back_end.connect(back_address.zmq_url())

        def callback(from_name, to_name, zmq_stream, msgs):
            log.debug("Routing from {0} to {1} messages {2}"
                      .format(from_name, to_name, msgs))
            zmq_stream.send_multipart(msgs)
            zmq_stream.flush()

        self._front_end.on_recv(lambda msgs:
                                callback(front_end_name, back_end_name, self._back_end, msgs))
        self._back_end.on_recv(lambda msgs:
                               callback(back_end_name, front_end_name, self._front_end, msgs))

# 需要导入模块: from zmq.eventloop.zmqstream import ZMQStream [as 别名]
# 或者: from zmq.eventloop.zmqstream.ZMQStream import connect [as 别名]
    def stream(self, sock_type, sock_addr, sock_bind, 
            callback=None, subscribe=''):

        assert self.ctx is not None

        sock_addr = sock_addr % {
                'port': random.randint(1024,65535),
                }

        s = ZMQStream(
                self.ctx.socket(sock_type))

        if sock_type == zmq.SUB:
            s.setsockopt(zmq.SUBSCRIBE, subscribe)

        if sock_bind:
            s.bind(sock_addr)
        else:
            s.connect(sock_addr)

        if callback:
            s.on_recv(callback)

        return (s, sock_addr)

# 需要导入模块: from zmq.eventloop.zmqstream import ZMQStream [as 别名]
# 或者: from zmq.eventloop.zmqstream.ZMQStream import connect [as 别名]
class ZmqSubscriber(object):
    def __init__(self,
                 moduleName,
                 centralHost=SUBSCRIBER_OPT_DEFAULTS['centralHost'],
                 context=None,
                 centralPublishEndpoint=SUBSCRIBER_OPT_DEFAULTS['centralPublishEndpoint'],
                 replay=None):
        self.moduleName = moduleName
        self.centralHost = centralHost

        if context is None:
            context = zmq.Context.instance()
        self.context = context

        self.centralPublishEndpoint = parseEndpoint(centralPublishEndpoint,
                                                    defaultPort=DEFAULT_CENTRAL_PUBLISH_PORT,
                                                    centralHost=self.centralHost)
        self.replayPaths = replay
        if self.replayPaths is None:
            self.replayPaths = []

        self.handlers = {}
        self.counter = 0
        self.deserializer = serializers.get_deserializer('json')
        self.stream = None

    @classmethod
    def addOptions(cls, parser, defaultModuleName):
        if not parser.has_option('--centralHost'):
            parser.add_option('--centralHost',
                              default=SUBSCRIBER_OPT_DEFAULTS['centralHost'],
                              help='Host where central runs [%default]')
        if not parser.has_option('--moduleName'):
            parser.add_option('--moduleName',
                              default=defaultModuleName,
                              help='Name to use for this module [%default]')
        if not parser.has_option('--centralPublishEndpoint'):
            parser.add_option('--centralPublishEndpoint',
                              default=SUBSCRIBER_OPT_DEFAULTS['centralPublishEndpoint'],
                              help='Endpoint where central publishes messages [%default]')
        if not parser.has_option('--replay'):
            parser.add_option('--replay',
                              action='append',
                              help='Replay specified message log (can specify multiple times), or use - to read from stdin')

    @classmethod
    def getOptionValues(cls, opts):
        result = {}
        for key in SUBSCRIBER_OPT_DEFAULTS.iterkeys():
            val = getattr(opts, key, None)
            if val is not None:
                result[key] = val
        return result

    def start(self):
        sock = self.context.socket(zmq.SUB)
        self.stream = ZMQStream(sock)
        # causes problems with multiple instances
        #self.stream.setsockopt(zmq.IDENTITY, self.moduleName)
        self.stream.connect(self.centralPublishEndpoint)
        logging.info('zmq.subscriber: connected to central at %s', self.centralPublishEndpoint)
        self.stream.on_recv(self.routeMessages)

    def routeMessages(self, messages):
        for msg in messages:
            self.routeMessage(msg)

    def routeMessage(self, msg):
        colonIndex = msg.find(':')
        topic = msg[:(colonIndex + 1)]
        body = msg[(colonIndex + 1):]

        handled = 0
        for topicPrefix, registry in self.handlers.iteritems():
            if topic.startswith(topicPrefix):
                for handler in registry.itervalues():
                    handler(topic[:-1], body)
                    handled = 1

        return handled

    def subscribeRaw(self, topicPrefix, handler):
        topicRegistry = self.handlers.setdefault(topicPrefix, {})
        if not topicRegistry:
            logging.info('zmq.subscriber: subscribe %s', topicPrefix)
            self.stream.setsockopt(zmq.SUBSCRIBE, topicPrefix)
        handlerId = (topicPrefix, self.counter)
        topicRegistry[self.counter] = handler
        self.counter += 1
        return handlerId

    def subscribeJson(self, topicPrefix, handler):
        def jsonHandler(topicPrefix, body):
            return handler(topicPrefix, convertToDotDictRecurse(json.loads(body)))
        return self.subscribeRaw(topicPrefix, jsonHandler)

    def subscribeDjango(self, topicPrefix, handler):
        def djangoHandler(topicPrefix, body):
            obj = json.loads(body)
            dataText = json.dumps([obj['data']])
#.........这里部分代码省略.........

# 需要导入模块: from zmq.eventloop.zmqstream import ZMQStream [as 别名]
# 或者: from zmq.eventloop.zmqstream.ZMQStream import connect [as 别名]
class MDPWorker(object):

    """Class for the MDP worker side.

    Thin encapsulation of a zmq.XREQ socket.
    Provides a send method with optional timeout parameter.

    Will use a timeout to indicate a broker failure.
    """

    _proto_version = b'MDPW01'

    # TODO: integrate that into API
    HB_INTERVAL = 1000  # in milliseconds
    HB_LIVENESS = 3    # HBs to miss before connection counts as dead

    def __init__(self, context, endpoint, service):
        """Initialize the MDPWorker.

        context is the zmq context to create the socket from.
        service is a byte-string with the service name.
        """
        self.context = context
        self.endpoint = endpoint
        self.service = service
        self.stream = None
        self._tmo = None
        self.need_handshake = True
        self.ticker = None
        self._delayed_cb = None
        self._create_stream()
        return

    def _create_stream(self):
        """Helper to create the socket and the stream.
        """
        socket = self.context.socket(zmq.XREQ)
        ioloop = IOLoop.instance()
        self.stream = ZMQStream(socket, ioloop)
        self.stream.on_recv(self._on_message)
        self.stream.socket.setsockopt(zmq.LINGER, 0)
        self.stream.connect(self.endpoint)
        self.ticker = PeriodicCallback(self._tick, self.HB_INTERVAL)
        self._send_ready()
        self.ticker.start()
        return

    def _send_ready(self):
        """Helper method to prepare and send the workers READY message.
        """
        ready_msg = [ b'', self._proto_version, chr(1), self.service ]
        self.stream.send_multipart(ready_msg)
        self.curr_liveness = self.HB_LIVENESS
        return

    def _tick(self):
        """Method called every HB_INTERVAL milliseconds.
        """
        self.curr_liveness -= 1
##         print '%.3f tick - %d' % (time.time(), self.curr_liveness)
        self.send_hb()
        if self.curr_liveness >= 0:
            return
        print '%.3f lost connection' % time.time()
        # ouch, connection seems to be dead
        self.shutdown()
        # try to recreate it
        self._delayed_cb = DelayedCallback(self._create_stream, 5000)
        self._delayed_cb.start()
        return

    def send_hb(self):
        """Construct and send HB message to broker.
        """
        msg = [ b'', self._proto_version, chr(4) ]
        self.stream.send_multipart(msg)
        return

    def shutdown(self):
        """Method to deactivate the worker connection completely.

        Will delete the stream and the underlying socket.
        """
        if self.ticker:
            self.ticker.stop()
            self.ticker = None
        if not self.stream:
            return
        self.stream.socket.close()
        self.stream.close()
        self.stream = None
        self.timed_out = False
        self.need_handshake = True
        self.connected = False
        return

    def reply(self, msg):
        """Send the given message.

        msg can either be a byte-string or a list of byte-strings.
#.........这里部分代码省略.........

# 需要导入模块: from zmq.eventloop.zmqstream import ZMQStream [as 别名]
# 或者: from zmq.eventloop.zmqstream.ZMQStream import connect [as 别名]
class Subscriber (object):
    def __init__(self,
            sub_uri=defaults.subscriber_sub_uri,
            patterns=None,
            callbacks=None,
            ):

        self.sub_uri = sub_uri
        
        if patterns:
            self.patterns = patterns
        else:
            self.patterns = []

        if callbacks:
            self.callbacks = callbacks
        else:
            self.callbacks = []
        
        self.setup_logging()
        self.setup_zmq()
        self.setup_sockets()
        self.setup_subscriptions()
        self.setup_events()

    def setup_logging(self):
        self.log = logging.getLogger('zmqevt.subscriber')

    def setup_zmq(self):
        self.context = zmq.Context()

    def setup_sockets(self):
        self.sub = ZMQStream(self.context.socket(zmq.SUB))
        self.sub.connect(self.sub_uri)

    def setup_subscriptions(self):
        if self.patterns:
            for p in self.patterns:
                self.subscribe(p)

    def subscribe(self, pattern):
        self.log.debug('Subcribe to "%s".' % pattern)
        self.sub.setsockopt(zmq.SUBSCRIBE, pattern)

    def setup_events(self):
        self.sub.on_recv(self.on_recv)

    def register_callback(self, func, data=None):
        self.callbacks.append((func,data))

    def unregister_callback(self, func):
        self.callbacks = [x for x in self.callbacks if x[0] is not func]

    def on_recv(self, msg):
        self.log.debug('Receive: %s' % (str(msg)))

        assert len(msg) == 2, 'Received invalid message.'

        # This regenerates the event to ensure that we don't
        # pass on invalid data.
        try:
            evt = event.Event.load(msg)
        except Exception, detail:
            self.log.error('Error processing message: %s' % detail)
            return

        self.log.debug('Event: %s' % (str(evt.dump())))

        for func, data in self.callbacks:
            func(evt, data=data)

# 需要导入模块: from zmq.eventloop.zmqstream import ZMQStream [as 别名]
# 或者: from zmq.eventloop.zmqstream.ZMQStream import connect [as 别名]
class ZmqSubscriber(object):
    def __init__(self,
                 moduleName,
                 context=None,
                 centralPublishEndpoint=SUBSCRIBER_OPT_DEFAULTS['centralPublishEndpoint']):
        self.moduleName = moduleName

        if context is None:
            context = zmq.Context.instance()
        self.context = context

        self.centralPublishEndpoint = parseEndpoint(centralPublishEndpoint,
                                                    defaultPort=DEFAULT_CENTRAL_PUBLISH_PORT)

        self.handlers = {}
        self.counter = 0
        self.deserializer = serializers.get_deserializer('json')

    @classmethod
    def addOptions(cls, parser, defaultModuleName):
        if not parser.has_option('--moduleName'):
            parser.add_option('--moduleName',
                              default=defaultModuleName,
                              help='Name to use for this module [%default]')
        if not parser.has_option('--centralPublishEndpoint'):
            parser.add_option('--centralPublishEndpoint',
                              default=SUBSCRIBER_OPT_DEFAULTS['centralPublishEndpoint'],
                              help='Endpoint where central publishes messages [%default]')

    @classmethod
    def getOptionValues(cls, opts):
        result = {}
        for key in SUBSCRIBER_OPT_DEFAULTS.iterkeys():
            val = getattr(opts, key, None)
            if val is not None:
                result[key] = val
        return result

    def start(self):
        sock = self.context.socket(zmq.SUB)
        self.stream = ZMQStream(sock)
        self.stream.setsockopt(zmq.IDENTITY, self.moduleName)
        self.stream.connect(self.centralPublishEndpoint)
        self.stream.on_recv(self.routeMessage)

    def routeMessage(self, messages):
        for msg in messages:
            colonIndex = msg.find(':')
            topic = msg[:(colonIndex + 1)]
            body = msg[(colonIndex + 1):]

            for topicPrefix, registry in self.handlers.iteritems():
                if topic.startswith(topicPrefix):
                    topicRegistry = registry
                    break

            for handler in topicRegistry.itervalues():
                handler(topic[:-1], body)

    def subscribeRaw(self, topic, handler):
        topicRegistry = self.handlers.setdefault(topic, {})
        if not topicRegistry:
            self.stream.setsockopt(zmq.SUBSCRIBE, topic)
        handlerId = (topic, self.counter)
        topicRegistry[self.counter] = handler
        self.counter += 1
        return handlerId

    def subscribeJson(self, topic, handler):
        def jsonHandler(topic, body):
            return handler(topic, json.loads(body))
        return self.subscribeRaw(topic, jsonHandler)

    def subscribeDjango(self, topic, handler):
        def djangoHandler(topic, body):
            obj = json.loads(body)
            dataText = json.dumps([obj['data']])
            modelInstance = self.deserializer(dataText)[0]
            return handler(topic, modelInstance)
        return self.subscribeRaw(topic, djangoHandler)

    def unsubscribe(self, handlerId):
        topic, index = handlerId
        topicRegistry = self.handlers[topic]
        del topicRegistry[index]
        if not topicRegistry:
            self.stream.setsockopt(zmq.UNSUBSCRIBE, topic)

    def connect(self, endpoint):
        self.stream.connect(endpoint)

# 需要导入模块: from zmq.eventloop.zmqstream import ZMQStream [as 别名]
# 或者: from zmq.eventloop.zmqstream.ZMQStream import connect [as 别名]
    def init_hub(self):
        """construct"""
        client_iface = "%s://%s:" % (self.client_transport, self.client_ip) + "%i"
        engine_iface = "%s://%s:" % (self.engine_transport, self.engine_ip) + "%i"

        ctx = self.context
        loop = self.loop

        # Registrar socket
        q = ZMQStream(ctx.socket(zmq.ROUTER), loop)
        q.bind(client_iface % self.regport)
        self.log.info("Hub listening on %s for registration.", client_iface % self.regport)
        if self.client_ip != self.engine_ip:
            q.bind(engine_iface % self.regport)
            self.log.info("Hub listening on %s for registration.", engine_iface % self.regport)

        ### Engine connections ###

        # heartbeat
        hpub = ctx.socket(zmq.PUB)
        hpub.bind(engine_iface % self.hb[0])
        hrep = ctx.socket(zmq.ROUTER)
        hrep.bind(engine_iface % self.hb[1])
        self.heartmonitor = HeartMonitor(loop=loop, config=self.config, log=self.log,
                                pingstream=ZMQStream(hpub,loop),
                                pongstream=ZMQStream(hrep,loop)
                            )

        ### Client connections ###
        # Notifier socket
        n = ZMQStream(ctx.socket(zmq.PUB), loop)
        n.bind(client_iface%self.notifier_port)

        ### build and launch the queues ###

        # monitor socket
        sub = ctx.socket(zmq.SUB)
        sub.setsockopt(zmq.SUBSCRIBE, b"")
        sub.bind(self.monitor_url)
        sub.bind('inproc://monitor')
        sub = ZMQStream(sub, loop)

        # connect the db
        self.log.info('Hub using DB backend: %r'%(self.db_class.split()[-1]))
        # cdir = self.config.Global.cluster_dir
        self.db = import_item(str(self.db_class))(session=self.session.session,
                                            config=self.config, log=self.log)
        time.sleep(.25)
        try:
            scheme = self.config.TaskScheduler.scheme_name
        except AttributeError:
            from .scheduler import TaskScheduler
            scheme = TaskScheduler.scheme_name.get_default_value()
        # build connection dicts
        self.engine_info = {
            'control' : engine_iface%self.control[1],
            'mux': engine_iface%self.mux[1],
            'heartbeat': (engine_iface%self.hb[0], engine_iface%self.hb[1]),
            'task' : engine_iface%self.task[1],
            'iopub' : engine_iface%self.iopub[1],
            # 'monitor' : engine_iface%self.mon_port,
            }

        self.client_info = {
            'control' : client_iface%self.control[0],
            'mux': client_iface%self.mux[0],
            'task' : (scheme, client_iface%self.task[0]),
            'iopub' : client_iface%self.iopub[0],
            'notification': client_iface%self.notifier_port
            }
        self.log.debug("Hub engine addrs: %s", self.engine_info)
        self.log.debug("Hub client addrs: %s", self.client_info)

        # resubmit stream
        r = ZMQStream(ctx.socket(zmq.DEALER), loop)
        url = util.disambiguate_url(self.client_info['task'][-1])
        r.setsockopt(zmq.IDENTITY, self.session.bsession)
        r.connect(url)

        self.hub = Hub(loop=loop, session=self.session, monitor=sub, heartmonitor=self.heartmonitor,
                query=q, notifier=n, resubmit=r, db=self.db,
                engine_info=self.engine_info, client_info=self.client_info,
                log=self.log)

# 需要导入模块: from zmq.eventloop.zmqstream import ZMQStream [as 别名]
# 或者: from zmq.eventloop.zmqstream.ZMQStream import connect [as 别名]
class Worker(object):

    """Class for the MDP worker side.

    Thin encapsulation of a zmq.DEALER socket.
    Provides a send method with optional timeout parameter.

    Will use a timeout to indicate a broker failure.
    """
    max_forks = 10

    ipc = 'ipc:///tmp/zmq-rpc-'+str(uuid4())
    HB_INTERVAL = 1000  # in milliseconds
    HB_LIVENESS = 3    # HBs to miss before connection counts as dead

    def __init__(self, context, endpoint, service, multicasts=()):
        """Initialize the MDPWorker.

        :param context:    is the zmq context to create the socket from
        :type context:     zmq.Context
        :param service:    service name - you can put hostname here
        :type service:     str
        :param multicasts: list of groups to subscribe
        :type multicasts:  list
        """
        self.context = context
        self.endpoint = endpoint
        self.service = service.encode('utf-8')  # convert to byte-string - required in python 3
        self.multicasts = [m.encode('utf-8') for m in multicasts]  # convert to byte-string
        self.stream = None
        self._tmo = None
        self.need_handshake = True
        self.ticker = None
        self._delayed_cb = None
        self._create_stream()
        self.forks = []
        self.curr_liveness = self.HB_LIVENESS

        socket = self.context.socket(zmq.ROUTER)
        socket.bind(self.ipc)
        self.stream_w = ZMQStream(socket)
        self.stream_w.on_recv(self._on_fork_response)
        self.reply_socket = None
        return

    def _create_stream(self):
        """Helper to create the socket and the stream.
        """
        self.on_log_event('broker.connect', 'Trying to connect do broker')
        socket = self.context.socket(zmq.DEALER)
        ioloop = IOLoop.instance()
        self.stream = ZMQStream(socket, ioloop)
        self.stream.on_recv(self._on_message)
        self.stream.socket.setsockopt(zmq.LINGER, 0)
        self.stream.connect(self.endpoint)
        self.ticker = PeriodicCallback(self._tick, self.HB_INTERVAL)
        self._send_ready()
        for m in self.multicasts:
            self._register_worker_to_multicast(m)
        self.ticker.start()
        return

    def _tick(self):
        """Method called every HB_INTERVAL milliseconds.
        """
        self.curr_liveness -= 1
        self.send_hb()
        if self.curr_liveness >= 0:
            return
        # ouch, connection seems to be dead
        self.on_log_event('broker.timeout', 'Connection to broker timeouted, disconnecting')
        self.shutdown(False)
        # try to recreate it
        self._delayed_cb = DelayedCallback(self._create_stream, 5000)
        self._delayed_cb.start()
        return

    def send_hb(self):
        """Construct and send HB message to broker.
        """
        msg = [b'', MDP_WORKER_VERSION, b'\x05']
        self.stream.send_multipart(msg)
        return

    def shutdown(self, final=True):
        """Method to deactivate the worker connection completely.

        Will delete the stream and the underlying socket.

        :param final: if shutdown is final and we want to close all sockets
        :type final:  bool
        """

        if self.ticker:
            self.ticker.stop()
            self.ticker = None
        if not self.stream:
            return

        self.stream.on_recv(None)
#.........这里部分代码省略.........

# 需要导入模块: from zmq.eventloop.zmqstream import ZMQStream [as 别名]
# 或者: from zmq.eventloop.zmqstream.ZMQStream import connect [as 别名]
class MQRep(object):

    """Class for the MDP worker side.

    Thin encapsulation of a zmq.DEALER socket.
    Provides a send method with optional timeout parameter.

    Will use a timeout to indicate a broker failure.
    """

    _proto_version = b'MDPW01'

    # TODO: integrate that into API
    HB_INTERVAL = 1000  # in milliseconds
    HB_LIVENESS = 3    # HBs to miss before connection counts as dead

    def __init__(self, context, service):
        """Initialize the MDPWorker.

        context is the zmq context to create the socket from.
        service is a byte-string with the service name.
        """
        if DEBUG:
            print("MQRep > __init__")
        cfg = Loader('mq').load()
        config = dict(cfg[1])
        if config['ip'].strip() == "*":
            config['ip'] = get_ip()
        self.endpoint = "tcp://{0}:{1}".format(config['ip'], config['req_rep_port'])
        self.context = context
        self.service = service
        self.stream = None
        self._tmo = None
        self.need_handshake = True
        self.ticker = None
        self._delayed_cb = None
        self._create_stream()

        ### patch fritz
        self._reconnect_in_progress = False
        ### end patch fritz
        return

    def _create_stream(self):
        """Helper to create the socket and the stream.
        """
        if DEBUG:
            print("MQRep > _create_stream")
        socket = ZmqSocket(self.context, zmq.DEALER)
        ioloop = IOLoop.instance()
        self.stream = ZMQStream(socket, ioloop)
        self.stream.on_recv(self._on_mpd_message)
        self.stream.socket.setsockopt(zmq.LINGER, 0)
        self.stream.connect(self.endpoint)
        if self.ticker != None:
            if DEBUG:
                print("MQRep > _create_stream - stop ticker")
            self.ticker.stop()
        self.ticker = PeriodicCallback(self._tick, self.HB_INTERVAL)
        self._send_ready()
        self.ticker.start()
        return

    def _send_ready(self):
        """Helper method to prepare and send the workers READY message.
        """
        if DEBUG:
            print("MQREP > _send_ready")
        ready_msg = [ b'', self._proto_version, b'\x01', self.service ]
        self.stream.send_multipart(ready_msg)
        self.curr_liveness = self.HB_LIVENESS
        if DEBUG:
            print("MQREP > _send_ready > curr_liveness <= {0}".format(self.HB_LIVENESS))
        return

    def _tick(self):
        """Method called every HB_INTERVAL milliseconds.
        """
        if DEBUG:
            print("MQREP > _tick")
        self.curr_liveness -= 1
        if DEBUG:
            print('MQREP > _tick - {0} tick = {1}'.format(time.time(), self.curr_liveness))
        self.send_hb()
        if self.curr_liveness >= 0:
            return
        if DEBUG:
            print('MQREP > _tick - {0} lost connection'.format(time.time()))
        # ouch, connection seems to be dead
        self.shutdown()
        # try to recreate it
        self._delayed_cb = DelayedCallback(self._create_stream, self.HB_INTERVAL)
        self._delayed_cb.start()
        return

    def send_hb(self):
        """Construct and send HB message to broker.
        """
        msg = [ b'', self._proto_version, b'\x04' ]
        self.stream.send_multipart(msg)
#.........这里部分代码省略.........

# 需要导入模块: from zmq.eventloop.zmqstream import ZMQStream [as 别名]
# 或者: from zmq.eventloop.zmqstream.ZMQStream import connect [as 别名]
class ZmqPublisher(object):
    def __init__(self,
                 moduleName,
                 centralHost=PUBLISHER_OPT_DEFAULTS['centralHost'],
                 context=None,
                 centralSubscribeEndpoint=PUBLISHER_OPT_DEFAULTS['centralSubscribeEndpoint'],
                 publishEndpoint=PUBLISHER_OPT_DEFAULTS['publishEndpoint'],
                 heartbeatPeriodMsecs=PUBLISHER_OPT_DEFAULTS['heartbeatPeriodMsecs'],
                 # highWaterMark=PUBLISHER_OPT_DEFAULTS['highWaterMark']
                 ):
        self.moduleName = moduleName
        self.centralHost = centralHost

        if context is None:
            context = zmq.Context.instance()
        self.context = context

        self.centralSubscribeEndpoint = parseEndpoint(centralSubscribeEndpoint,
                                                      defaultPort=DEFAULT_CENTRAL_SUBSCRIBE_PORT,
                                                      centralHost=self.centralHost)
        self.publishEndpoint = parseEndpoint(publishEndpoint,
                                             defaultPort='random')
        self.heartbeatPeriodMsecs = heartbeatPeriodMsecs
        #self.highWaterMark = highWaterMark

        self.pubStream = None
        self.heartbeatTimer = None

        self.serializer = serializers.get_serializer('json')()

    @classmethod
    def addOptions(cls, parser, defaultModuleName):
        if not parser.has_option('--centralHost'):
            parser.add_option('--centralHost',
                              default=PUBLISHER_OPT_DEFAULTS['centralHost'],
                              help='Host where central runs [%default]')
        if not parser.has_option('--moduleName'):
            parser.add_option('--moduleName',
                              default=defaultModuleName,
                              help='Name to use for this module [%default]')
        if not parser.has_option('--centralSubscribeEndpoint'):
            parser.add_option('--centralSubcribeEndpoint',
                              default=PUBLISHER_OPT_DEFAULTS['centralSubscribeEndpoint'],
                              help='Endpoint where central listens for messages [%default]')
        if not parser.has_option('--publishEndpoint'):
            parser.add_option('--publishEndpoint',
                              default=PUBLISHER_OPT_DEFAULTS['publishEndpoint'],
                              help='Endpoint to publish messages on [%default]')
        if not parser.has_option('--heartbeatPeriodMsecs'):
            parser.add_option('--heartbeatPeriodMsecs',
                              default=PUBLISHER_OPT_DEFAULTS['heartbeatPeriodMsecs'],
                              type='int',
                              help='Period for sending heartbeats to central [%default]')
        #if not parser.has_option('--highWaterMark'):
        #    parser.add_option('--highWaterMark',
        #                      default=PUBLISHER_OPT_DEFAULTS['highWaterMark'],
        #                      type='int',
        #                      help='High-water mark for publish socket (see 0MQ docs) [%default]')

    @classmethod
    def getOptionValues(cls, opts):
        result = {}
        for key in PUBLISHER_OPT_DEFAULTS.iterkeys():
            val = getattr(opts, key, None)
            if val is not None:
                result[key] = val
        return result

    def heartbeat(self):
        logging.debug('ZmqPublisher: heartbeat')
        self.sendJson('central.heartbeat.%s' % self.moduleName,
                      {'host': getShortHostName(),
                       'pub': self.publishEndpoint})

    def sendRaw(self, topic, body):
        self.pubStream.send('%s:%s' % (topic, body))
        self.pubStream.flush()

    def sendJson(self, topic, obj):
        if isinstance(obj, dict):
            obj.setdefault('module', self.moduleName)
            obj.setdefault('timestamp', str(getTimestamp()))
        self.sendRaw(topic, json.dumps(obj))

    def sendDjango(self, modelInstance, topic=None, topicSuffix=None):
        dataText = self.serializer.serialize([modelInstance])
        data = json.loads(dataText)[0]
        if topic is None:
            topic = data['model'].encode('utf-8')
            if topicSuffix is not None:
                topic += topicSuffix
        self.sendJson(topic, {'data': data})

    def start(self):
        pubSocket = self.context.socket(zmq.PUB)
        self.pubStream = ZMQStream(pubSocket)
        # self.pubStream.setsockopt(zmq.IDENTITY, self.moduleName)
        # self.pubStream.setsockopt(zmq.HWM, self.highWaterMark)
        self.pubStream.connect(self.centralSubscribeEndpoint)
        logging.info('zmq.publisher: connected to central at %s', self.centralSubscribeEndpoint)
#.........这里部分代码省略.........

# 需要导入模块: from zmq.eventloop.zmqstream import ZMQStream [as 别名]
# 或者: from zmq.eventloop.zmqstream.ZMQStream import connect [as 别名]
class ZmqCentral(object):
    def __init__(self, opts):
        self.opts = opts
        self.info = {}

    def announceConnect(self, moduleName, params):
        logging.info('module %s connected', moduleName)
        self.injectStream.send('central.connect.%s:%s'
                               % (moduleName, json.dumps(params)))

    def announceDisconnect(self, moduleName):
        logging.info('module %s disconnected', moduleName)
        self.injectStream.send('central.disconnect.%s:%s'
                               % (moduleName,
                                  json.dumps({'timestamp': str(getTimestamp())})))

    def logMessage(self, msg, posixTime=None, attachmentDir='-'):
        mlog = self.messageLog
        mlog.write('@@@ %d %d %s ' % (getTimestamp(posixTime), len(msg), attachmentDir))
        mlog.write(msg)
        mlog.write('\n')

    def logMessageWithAttachments0(self, msg):
        parsed = parseMessage(msg)
        posixTime = time.time()

        # construct attachment directory
        dt = datetime.datetime.utcfromtimestamp(posixTime)
        dateText = dt.strftime('%Y-%m-%d')
        timeText = dt.strftime('%H-%M-%S') + '.%06d' % dt.microsecond
        uniq = '%08x' % random.getrandbits(32)
        attachmentSuffix = os.path.join('attachments',
                                        dateText,
                                        timeText,
                                        parsed['topic'],
                                        uniq)
        attachmentPath = os.path.join(self.logDir, attachmentSuffix)
        os.makedirs(attachmentPath)

        # write attachments to attachment directory
        for attachment in parsed['attachments']:
            fullName = os.path.join(attachmentPath, attachment.get_filename())
            open(fullName, 'wb').write(attachment.get_payload())

        # log message with a pointer to the attachment directory
        self.logMessage(':'.join((parsed['topic'], parsed['json'])),
                        posixTime,
                        attachmentSuffix)

    def logMessageWithAttachments(self, msg):
        try:
            return self.logMessageWithAttachments0(msg)
        except:  # pylint: disable=W0702
            self.logException('logging message with attachments')

    def handleHeartbeat(self, params):
        moduleName = params['module'].encode('utf-8')
        now = getTimestamp()

        oldInfo = self.info.get(moduleName, None)
        if oldInfo:
            if oldInfo.get('pub', None) != params.get('pub', None):
                self.announceDisconnect(moduleName)
                self.announceConnect(moduleName, params)
        else:
            self.announceConnect(moduleName, params)

        self.info[moduleName] = params
        keepalive = params.get('keepalive', DEFAULT_KEEPALIVE_US)
        params['timeout'] = now + keepalive
        return 'ok'

    def handleInfo(self):
        return self.info

    def logException(self, whileClause):
        errClass, errObject, errTB = sys.exc_info()[:3]
        errText = '%s.%s: %s' % (errClass.__module__,
                                 errClass.__name__,
                                 str(errObject))
        logging.warning(''.join(traceback.format_tb(errTB)))
        logging.warning(errText)
        logging.warning('[error while %s at time %s]', whileClause, getTimestamp())

    def handleMessages(self, messages):
        for msg in messages:
            if hasAttachments(msg):
                self.logMessageWithAttachments(msg)
            else:
                self.logMessage(msg)
            if msg.startswith('central.heartbeat.'):
                try:
                    _topic, body = msg.split(':', 1)
                    self.handleHeartbeat(json.loads(body))
                except:  # pylint: disable=W0702
                    self.logException('handling heartbeat')

    def handleRpcCall(self, messages):
        for msg in messages:
            try:
#.........这里部分代码省略.........