Python kvmsg.KVMsg类代码示例

    def control_message (self):
        msg = self.pipe.recv_multipart()
        command = msg.pop(0)

        if command == "CONNECT":
            address = msg.pop(0)
            port = int(msg.pop(0))
            if len(self.servers) < SERVER_MAX:
                self.servers.append(CloneServer(self.ctx, address, port, self.subtree))
                self.publisher.connect("%s:%i" % (address,port+2))
            else:
                logging.error("E: too many servers (max. %i)", SERVER_MAX)
        elif command == "SET":
            key,value,sttl = msg
            ttl = int(sttl)

            # Send key-value pair on to server
            kvmsg = KVMsg(0, key=key, body=value)
            kvmsg.store(self.kvmap)
            if ttl:
                kvmsg["ttl"] = ttl
            kvmsg.send(self.publisher)
        elif command == "GET":
            key = msg[0]
            value = self.kvmap.get(key)
            self.pipe.send(value.body if value else '')

    def handle_snapshot(self, msg):
        """snapshot requests"""

        logging.info("I: received state request on snapshot: %s" % msg)
        if len(msg) != 3 or msg[1] != "ICANHAZ?":
            print "E: bad request, aborting"
            dump(msg)
            self.loop.stop()
            return
        identity, request, subtree = msg
        if subtree:
            # Send state snapshot to client
            route = Route(self.snapshot, identity, subtree)

            # For each entry in kvmap, send kvmsg to client
            for k, v in self.kvmap.items():
                logging.info("I: Sending snapshot: %s" % v)
                send_single(k, v, route)

            # Now send END message with sequence number
            self.snapshot.send(identity, zmq.SNDMORE)
            kvmsg = KVMsg(self.sequence)
            kvmsg.key = "KTHXBAI"
            kvmsg.body = subtree
            logging.info("I: Sending snapshot end message: %s" % kvmsg)
            kvmsg.send(self.snapshot)

 def handle_subscriber(self, msg):
     """Collect updates from peer (master)
     We're always slave when we get these updates
     """
     if self.master:
         logging.warn("received subscriber message, but we are master %s", msg)
         return
     
     # Get state snapshot if necessary
     if self.kvmap is None:
         self.kvmap = {}
         snapshot = self.ctx.socket(zmq.DEALER)
         snapshot.linger = 0
         snapshot.connect("tcp://localhost:%i" % self.peer)
         
         logging.info ("I: asking for snapshot from: tcp://localhost:%d",
                     self.peer)
         snapshot.send_multipart(["ICANHAZ?", ''])
         while True:
             try:
                 kvmsg = KVMsg.recv(snapshot)
             except KeyboardInterrupt:
                 # Interrupted
                 self.bstar.loop.stop()
                 return
             if kvmsg.key == "KTHXBAI":
                 self.sequence = kvmsg.sequence
                 break          # Done
             kvmsg.store(self.kvmap)
     
         logging.info ("I: received snapshot=%d", self.sequence)
     
     # Find and remove update off pending list
     kvmsg = KVMsg.from_msg(msg)
     # update integer ttl -> timestamp
     ttl = kvmsg.get('ttl')
     if ttl is not None:
         kvmsg['ttl'] = time.time() + ttl
     
     if kvmsg.key != "HUGZ":
         if not self.was_pending(kvmsg):
             # If master update came before client update, flip it
             # around, store master update (with sequence) on pending
             # list and use to clear client update when it comes later
             self.pending.append(kvmsg)
     
         # If update is more recent than our kvmap, apply it
         if (kvmsg.sequence > self.sequence):
             self.sequence = kvmsg.sequence
             kvmsg.store(self.kvmap)
             logging.info ("I: received update=%d", self.sequence)

 def handle_collect(self, msg):
     """Collect updates from clients"""
     kvmsg = KVMsg.from_msg(msg)
     self.sequence += 1
     kvmsg.sequence = self.sequence
     kvmsg.send(self.publisher)
     ttl = float(kvmsg.get('ttl', 0))
     if ttl:
         kvmsg['ttl'] = time.time() + ttl
     kvmsg.store(self.kvmap)
     logging.info("I: publishing update=%d", self.sequence)

 def handle_collect(self, msg):
     """Collect updates from clients"""
     kvmsg = KVMsg.from_msg(msg)
     logging.info("I: received state update on collector: %s" % kvmsg)
     self.sequence += 1
     kvmsg.sequence = self.sequence
     logging.info("I: publishing update: %s", kvmsg)
     kvmsg.send(self.publisher)
     ttl = kvmsg.get('ttl')
     if ttl is not None:
         kvmsg['ttl'] = time.time() + int(ttl)
     kvmsg.store(self.kvmap)

    def handle_snapshot(self, socket, msg):
        """snapshot requests"""
        if msg[1] != "ICANHAZ?" or len(msg) != 3:
            logging.error("E: bad request, aborting")
            dump(msg)
            self.bstar.loop.stop()
            return
        identity, request = msg[:2]
        if len(msg) >= 3:
            subtree = msg[2]
            # Send state snapshot to client
            route = Route(socket, identity, subtree)

            # For each entry in kvmap, send kvmsg to client
            for k,v in self.kvmap.items():
                send_single(k,v,route)
            
            # Now send END message with sequence number
            logging.info("I: Sending state shapshot=%d" % self.sequence)
            socket.send(identity, zmq.SNDMORE)
            kvmsg = KVMsg(self.sequence)
            kvmsg.key = "KTHXBAI"
            kvmsg.body = subtree
            kvmsg.send(socket)

 def handle_collect(self, msg):
     """Collect updates from clients
     
     If we're master, we apply these to the kvmap
     If we're slave, or unsure, we queue them on our pending list
     """
     kvmsg = KVMsg.from_msg(msg)
     if self.master:
         self.sequence += 1
         kvmsg.sequence = self.sequence
         kvmsg.send(self.publisher)
         ttl = kvmsg.get('ttl')
         if ttl is not None:
             kvmsg['ttl'] = time.time() + ttl
         kvmsg.store(self.kvmap)
         logging.info("I: publishing update=%d", self.sequence)
     else:
         # If we already got message from master, drop it, else
         # hold on pending list
         if not self.was_pending(kvmsg):
             self.pending.append(kvmsg)

 def send_hugz(self):
     """Send hugz to anyone listening on the publisher socket"""
     kvmsg = KVMsg(self.sequence)
     kvmsg.key = "HUGZ"
     kvmsg.body = ""
     kvmsg.send(self.publisher)

def main():
    
    # Prepare our context and subscriber
    ctx = zmq.Context()
    snapshot = ctx.socket(zmq.DEALER)
    snapshot.linger = 0
    snapshot.connect("tcp://localhost:5556")
    subscriber = ctx.socket(zmq.SUB)
    subscriber.linger = 0
    subscriber.setsockopt(zmq.SUBSCRIBE, SUBTREE)
    subscriber.connect("tcp://localhost:5557")
    publisher = ctx.socket(zmq.PUB)
    publisher.linger = 0
    publisher.connect("tcp://localhost:5558")

    random.seed(time.time())
    kvmap = {}

    # Get state snapshot
    sequence = 0
    snapshot.send_multipart(["ICANHAZ?", SUBTREE])
    while True:
        try:
            kvmsg = KVMsg.recv(snapshot)
        except:
            raise
            return          # Interrupted
            
        if kvmsg.key == "KTHXBAI":
            sequence = kvmsg.sequence
            print "I: Received snapshot=%d" % sequence
            break          # Done
        kvmsg.store(kvmap)
    
    poller = zmq.Poller()
    poller.register(subscriber, zmq.POLLIN)
    
    alarm = time.time()+1.
    while True:
        tickless = 1000*max(0, alarm - time.time())
        try:
            items = dict(poller.poll(tickless))
        except:
            break           # Interrupted
        
        if subscriber in items:
            kvmsg = KVMsg.recv(subscriber)

            # Discard out-of-sequence kvmsgs, incl. heartbeats
            if kvmsg.sequence > sequence:
                sequence = kvmsg.sequence
                kvmsg.store(kvmap)
                action = "update" if kvmsg.body else "delete"
                print "I: received %s=%d" % (action, sequence)
        
        # If we timed-out, generate a random kvmsg
        if time.time() >= alarm:
            kvmsg = KVMsg(0)
            kvmsg.key = SUBTREE + "%d" % random.randint(1,10000)
            kvmsg.body = "%d" % random.randint(1,1000000)
            kvmsg['ttl'] = random.randint(0,30)
            kvmsg.send(publisher)
            kvmsg.store(kvmap)
            alarm = time.time() + 1.

    print " Interrupted\n%d messages in" % sequence

def clone_agent(ctx, pipe):
    agent = CloneAgent(ctx, pipe)
    server = None

    while True:
        poller = zmq.Poller()
        poller.register(agent.pipe, zmq.POLLIN)
        poll_timer = None
        server_socket = None

        if agent.state == STATE_INITIAL:
            # In this state we ask the server for a snapshot,
            # if we have a server to talk to...
            if agent.servers:
                server = agent.servers[agent.cur_server]
                logging.info ("I: waiting for server at %s:%d...",
                    server.address, server.port)
                if (server.requests < 2):
                    server.snapshot.send_multipart(["ICANHAZ?", agent.subtree])
                    server.requests += 1
                server.expiry = time.time() + SERVER_TTL
                agent.state = STATE_SYNCING
                server_socket = server.snapshot
        elif agent.state == STATE_SYNCING:
            # In this state we read from snapshot and we expect
            # the server to respond, else we fail over.
            server_socket = server.snapshot
        elif agent.state == STATE_ACTIVE:
            # In this state we read from subscriber and we expect
            # the server to give hugz, else we fail over.
            server_socket = server.subscriber

        if server_socket:
            # we have a second socket to poll:
            poller.register(server_socket, zmq.POLLIN)

        if server is not None:
            poll_timer = 1e3 * max(0,server.expiry - time.time())

        # ------------------------------------------------------------
        # Poll loop
        try:
            items = dict(poller.poll(poll_timer))
        except:
            raise # DEBUG
            break # Context has been shut down

        if agent.pipe in items:
            agent.control_message()
        elif server_socket in items:
            kvmsg = KVMsg.recv(server_socket)

            # Anything from server resets its expiry time
            server.expiry = time.time() + SERVER_TTL
            if (agent.state == STATE_SYNCING):
                # Store in snapshot until we're finished
                server.requests = 0
                if kvmsg.key == "KTHXBAI":
                    agent.sequence = kvmsg.sequence
                    agent.state = STATE_ACTIVE
                    logging.info ("I: received from %s:%d snapshot=%d",
                        server.address, server.port, agent.sequence)
                else:
                    kvmsg.store(agent.kvmap)
            elif (agent.state == STATE_ACTIVE):
                # Discard out-of-sequence updates, incl. hugz
                if (kvmsg.sequence > agent.sequence):
                    agent.sequence = kvmsg.sequence
                    kvmsg.store(agent.kvmap)
                    action = "update" if kvmsg.body else "delete"

                    logging.info ("I: received from %s:%d %s=%d",
                        server.address, server.port, action, agent.sequence)
        else:
            # Server has died, failover to next
            logging.info ("I: server at %s:%d didn't give hugz",
                    server.address, server.port)
            agent.cur_server = (agent.cur_server + 1) % len(agent.servers)
            agent.state = STATE_INITIAL

def dkv_agent(ctx, pipe, connected_event):
    """ Asynchronous agent manages server pool and handles request/reply
    dialog when the application asks for it. """

    agent = DkvAgent(ctx, pipe, connected_event)
    server = None

    while True:
        poller = zmq.Poller()
        poller.register(agent.pipe, zmq.POLLIN)
        poll_timer = None
        server_socket = None

        if agent.state == agent.STATES.INITIAL:
            """In this state we ask the server for a snapshot,
            if we have a server to talk to..."""
            if agent.servers:
                server = agent.servers[agent.cur_server]

                logger.debug("Asking for snapshot from %s attempt=%d..", server.address, server.requests)

                if (server.requests < 2):
                    server.snapshot.send_multipart(["ICANHAZ?", agent.subtree])
                    server.requests += 1

                server.expiry = time.time() + SERVER_TTL
                agent.state = agent.STATES.SYNCING
                server_socket = server.snapshot

        elif agent.state == agent.STATES.SYNCING:
            """In this state we read from snapshot and we expect
            the server to respond, else we fail over."""
            server_socket = server.snapshot

        elif agent.state == agent.STATES.ACTIVE:
            """In this state we read from subscriber and we expect
            the server to give hugz, else we fail over."""
            server_socket = server.subscriber

        if server_socket:
            """we have a second socket to poll"""
            poller.register(server_socket, zmq.POLLIN)

        if server is not None:
            poll_timer = 1e3 * max(0, server.expiry - time.time())

        try:
            # Poll loop
            items = dict(poller.poll(poll_timer))
        except:
            raise  # DEBUG
            break  # Context has been shut down

        if agent.pipe in items:
            agent.control_message()

        elif server_socket in items:
            msg = server_socket.recv_multipart()
            #logger.debug('server_socket=%s, msg=%s', server_socket, msg)
            #logger.debug('msg=%s', msg)
            #kvmsg = KVMsg.recv(server_socket)
            kvmsg = KVMsg.from_msg(msg)
            #pp(kvmsg.__dict__)

            server.expiry = time.time() + SERVER_TTL    # Anything from server resets its expiry time

            if agent.state == agent.STATES.SYNCING:
                """Store in snapshot until we're finished"""
                server.requests = 0
                #logger.debug('Syncing state msg=%s', msg)
                if kvmsg.key == "KTHXBAI":
                    agent.sequence = kvmsg.sequence
                    agent.state = agent.STATES.ACTIVE
                    logger.info("Synced snapshot=%s from %s", agent.sequence, server.address)
                    logger.info("Connected to %s", server.address)
                    connected_event.set()
                else:
                    logger.debug("Syncing update=%s from %s", kvmsg.sequence, server.address)
                    kvmsg.store(agent.kvmap)

            elif agent.state == agent.STATES.ACTIVE:
                """Discard out-of-sequence updates, incl. hugz"""
                if kvmsg.sequence > agent.sequence:
                    agent.sequence = kvmsg.sequence
                    kvmsg.store(agent.kvmap)
                    action = "update" if kvmsg.body else "delete"

                    logger.debug("Received %s=%d from %s", action, agent.sequence, server.address)

                    """ Signal """
                    if kvmsg.key != 'HUGZ':  # Don't send signals if it's just hugz
                        Dkv.signals.on_sub.send(kvmsg, key=kvmsg.key, value=kvmsg.body, props=kvmsg.properties)

        else:
            """Server has died, failover to next"""
            if agent.state == agent.STATES.ACTIVE:
                level = logging.ERROR
                server_state = 'active'
            else:
                level = logging.WARNING
#.........这里部分代码省略.........

    def control_message(self):
        msg = self.pipe.recv_multipart()
        command = msg.pop(0)

        if self.debug:
            logger.debug('cmd=%s msg=%s', command, msg)
            pp(msg)

        if command == "CONNECT":
            #connect_kwargs = pipeline.load(msg[0])
            #self.connect(**connect_kwargs)
            self.connect(msg[0])
            self.pipe.send_multipart(['OK'])

        elif command == 'CONNECT_DISCOVERY':
            self.connect_via_discovery()
            self.pipe.send_multipart(['OK'])

        elif command == "DISCONNECT":
            address = msg.pop(0)
            port = int(msg.pop(0))
            self.disconnect(address, port)
            self.pipe.send_multipart(['OK'])

        elif command == "SET":
            #key, value, sttl, serializer = msg
            key, value, sttl = msg
            ttl = int(sttl)

            value = pipeline.load(value)

            # Send key-value pair on to server
            kvmsg = KVMsg(0, key=key, body=value)
            kvmsg.store(self.kvmap)
            if ttl:
                kvmsg["ttl"] = ttl
            #if serializer:
            #    kvmsg["serializer"] = serializer
            kvmsg.send(self.publisher)
            self.pipe.send_multipart(['OK'])

        elif command == "GET":
            key = msg[0]
            value = self.kvmap.get(key)

            if value:
                body = value.body
                #serializer = str(value.properties.get('serializer', ''))
            else:
                body = ''
                #serializer = ''

            body = pipeline.dump(body)

            #self.pipe.send_multipart([body, serializer])
            self.pipe.send_multipart([body])

        elif command == "SHOW":
            key = msg[0].upper()
            if key == 'SERVERS':
                self.pipe.send_multipart([
                    str(','.join([x.__repr__() for x in self.servers])), ''])
            elif key == 'SERVER':
                self.pipe.send_multipart([str(self.cur_server), ''])
            elif key == 'SEQ':
                self.pipe.send_multipart([str(self.sequence), ''])
            elif key == 'STATUS':
                self.pipe.send_multipart([str(self.cur_status), ''])
            elif key == 'KVMAP':
                #kvmap_s = pickle.dumps(self.kvmap)
                #self.pipe.send_multipart([kvmap_s, 'pickle'])
                kvmap_s = pipeline.dump(self.kvmap)
                self.pipe.send_multipart([kvmap_s])