Python _compat.xrange函数代码示例

def loop(rc, reset_last_key=None):
    """
    Regular debug loop that can be used to test how redis behaves during changes in the cluster.
    """
    if reset_last_key:
        rc.set("__last__", 0)

    last = False
    while last is False:
        try:
            last = rc.get("__last__")
            last = 0 if not last else int(last)
            print("starting at foo{0}".format(last))
        except Exception as e:
            print("error {0}".format(e))
            time.sleep(1)

    for i in xrange(last, 1000000000):  # noqa
        try:
            print("SET foo{0} {1}".format(i, i))
            rc.set("foo{0}".format(i), i)
            got = rc.get("foo{0}".format(i))
            print("GET foo{0} {1}".format(i, got))
            rc.set("__last__", i)
        except Exception as e:
            print("error {0}".format(e))

        time.sleep(0.05)

def timeit(rc, itterations=50000):
    """
    Time how long it take to run a number of set/get:s
    """
    t0 = time.time()
    for i in xrange(0, itterations):  # noqa
        s = "foo{0}".format(i)
        rc.set(s, i)
        rc.get(s)

    t1 = time.time() - t0
    print("{0}k SET/GET operations took: {1} seconds... {2} operations per second".format((itterations / 1000) * 2, t1, (itterations / t1) * 2))

 def rotate_slaves(self):
     "Round-robin slave balancer"
     slaves = self.sentinel_manager.discover_slaves(self.service_name)
     if slaves:
         if self.slave_rr_counter is None:
             self.slave_rr_counter = random.randint(0, len(slaves) - 1)
         for _ in xrange(len(slaves)):
             self.slave_rr_counter = (
                 self.slave_rr_counter + 1) % len(slaves)
             slave = slaves[self.slave_rr_counter]
             yield slave
     # Fallback to the master connection
     try:
         yield self.get_master_address()
     except MasterNotFoundError:
         pass
     raise SlaveNotFoundError('No slave found for %r' % (self.service_name))

    def read_response(self):
        response = self._buffer.readline()
        if not response:
            raise ConnectionError(SERVER_CLOSED_CONNECTION_ERROR)

        byte, response = byte_to_chr(response[0]), response[1:]

        if byte not in ('-', '+', ':', '$', '*'):
            raise InvalidResponse("Protocol Error: %s, %s" %
                                  (str(byte), str(response)))

        # server returned an error
        if byte == '-':
            response = nativestr(response)
            error = self.parse_error(response)
            # if the error is a ConnectionError, raise immediately so the user
            # is notified
            if isinstance(error, ConnectionError):
                raise error
            # otherwise, we're dealing with a ResponseError that might belong
            # inside a pipeline response. the connection's read_response()
            # and/or the pipeline's execute() will raise this error if
            # necessary, so just return the exception instance here.
            return error
        # single value
        elif byte == '+':
            pass
        # int value
        elif byte == ':':
            response = long(response)
        # bulk response
        elif byte == '$':
            length = int(response)
            if length == -1:
                return None
            response = self._buffer.read(length)
        # multi-bulk response
        elif byte == '*':
            length = int(response)
            if length == -1:
                return None
            response = [self.read_response() for i in xrange(length)]
        if isinstance(response, bytes) and self.encoding:
            response = response.decode(self.encoding)
            response = convertToNumber(response)
        return response

def timeit_pipeline(rc, itterations=50000):
    """
    Time how long it takes to run a number of set/get:s inside a cluster pipeline
    """
    t0 = time.time()
    for i in xrange(0, itterations):  # noqa
        s = "foo{0}".format(i)

        p = rc.pipeline()
        p.set(s, i)
        p.get(s)
        p.execute()

    t1 = time.time() - t0
    print("{0}k SET/GET operations inside pipelines took: {1} seconds... {2} operations per second".format(
        (itterations / 1000) * 2, t1, (itterations / t1) * 2)
    )

    def read_response(self):
        response = self.read()
        if not response:
            raise ConnectionError("Socket closed on remote end")

        byte, response = byte_to_chr(response[0]), response[1:]

        if byte not in ('-', '+', ':', '$', '*'):
            raise InvalidResponse("Protocol Error")

        # server returned an error
        if byte == '-':
            if nativestr(response).startswith('LOADING '):
                # if we're loading the dataset into memory, kill the socket
                # so we re-initialize (and re-SELECT) next time.
                raise ConnectionError("Redis is loading data into memory")
            # if the error starts with ERR, trim that off
            if nativestr(response).startswith('ERR '):
                response = response[4:]
            # *return*, not raise the exception class. if it is meant to be
            # raised, it will be at a higher level.
            return ResponseError(response)
        # single value
        elif byte == '+':
            pass
        # int value
        elif byte == ':':
            response = long(response)
        # bulk response
        elif byte == '$':
            length = int(response)
            if length == -1:
                return None
            response = self.read(length)
        # multi-bulk response
        elif byte == '*':
            length = int(response)
            if length == -1:
                return None
            response = [self.read_response() for i in xrange(length)]
        if isinstance(response, bytes) and self.encoding:
            response = response.decode(self.encoding)
        return response

    def _execute_pipeline(self, connection, commands, raise_on_error):
        """
        Code borrowed from StrictRedis so it can be fixed
        """
        # build up all commands into a single request to increase network perf
        if self.explicit_transaction:
            # build up all commands into a single request to increase network perf
            cmds = chain([(('MULTI', ), {})], commands, [(('EXEC', ), {})])
            all_cmds = connection.pack_commands([args for args, _ in cmds])
        else:
            all_cmds = connection.pack_commands([args for args, _ in commands])

        try:
            connection.send_packed_command(all_cmds)
        except ConnectionError as e:
            return [e for _ in xrange(len(commands))]  # noqa

        response = []

        if self.explicit_transaction:
            # Get rid of multi values (OK, QUEUED...)
            self.parse_response(connection, "_")
            for _, _ in enumerate(commands):
                self.parse_response(connection, "_")

            response = self.parse_response(connection, "_")
            if response is None:
                raise WatchError("Watched variable changed.")
            for r, cmd in izip(response, commands):
                response.append(r)
            self.explicit_transaction = False
            self.watching = False
        else:
            for args, options in commands:
                try:
                    response.append(self.parse_response(connection, args[0], **options))
                except (ConnectionError, ResponseError):
                    response.append(sys.exc_info()[1])

        if raise_on_error:
            self.raise_first_error(commands, response)

        return response

    def read_response(self):
        response = self.read()
        if not response:
            raise ConnectionError("Socket closed on remote end")

        byte, response = byte_to_chr(response[0]), response[1:]

        if byte not in ("-", "+", ":", "$", "*"):
            raise InvalidResponse("Protocol Error")

        # server returned an error
        if byte == "-":
            response = nativestr(response)
            if response.startswith("LOADING "):
                # if we're loading the dataset into memory, kill the socket
                # so we re-initialize (and re-SELECT) next time.
                raise ConnectionError("Redis is loading data into memory")
            # *return*, not raise the exception class. if it is meant to be
            # raised, it will be at a higher level.
            return self.parse_error(response)
        # single value
        elif byte == "+":
            pass
        # int value
        elif byte == ":":
            response = long(response)
        # bulk response
        elif byte == "$":
            length = int(response)
            if length == -1:
                return None
            response = self.read(length)
        # multi-bulk response
        elif byte == "*":
            length = int(response)
            if length == -1:
                return None
            response = [self.read_response() for i in xrange(length)]
        if isinstance(response, bytes) and self.encoding:
            response = response.decode(self.encoding)
        return response

    def _execute_pipeline(self, connection, commands, raise_on_error):
        """
        Code borrowed from StrictRedis so it can be fixed
        """
        # build up all commands into a single request to increase network perf
        all_cmds = connection.pack_commands([args for args, _ in commands])
        try:
            connection.send_packed_command(all_cmds)
        except ConnectionError as e:
            return [e for _ in xrange(len(commands))]  # noqa

        response = []
        for args, options in commands:
            try:
                response.append(self.parse_response(connection, args[0], **options))
            except (ConnectionError, ResponseError):
                response.append(sys.exc_info()[1])

        if raise_on_error:
            self.raise_first_error(commands, response)
        return response

    def read_response(self):
        response = self.read()
        if not response:
            raise ConnectionError("Socket closed on remote end")

        byte, response = byte_to_chr(response[0]), response[1:]

        if byte not in ('-', '+', ':', '$', '*'):
            raise InvalidResponse("Protocol Error")

        # server returned an error
        if byte == '-':
            response = nativestr(response)
            # *return*, not raise the exception class. if it is meant to be
            # raised, it will be at a higher level.
            return self.parse_error(response)
        # single value
        elif byte == '+':
            pass
        # int value
        elif byte == ':':
            response = long(response)
        # bulk response
        elif byte == '$':
            length = int(response)
            if length == -1:
                return None
            response = self.read(length)
        # multi-bulk response
        elif byte == '*':
            length = int(response)
            if length == -1:
                return None
            response = [self.read_response() for i in xrange(length)]
        if isinstance(response, bytes) and self.encoding:
            response = response.decode(self.encoding)
        return response

    def read_response(self):
        '''
        Reads one line from the wire, and interprets it.
        Example: the acknowledgment to an unsubscribe
        from topic myTopic on the wire looks like this:
        
             *3\r\n$11\r\nUNSUBSCRIBE\r\n$7\r\nmyTopic\r\n:1\r\n'
             
        *3    # three items to follow
        $11   # string of 11 chars
        UNSUBSCRIBE
        $7    # string of 7 chars
        myTopic
        :1    # one topic subscribed to now
        
        Each line will cause a recursive call to this method
        (see elif byte == '*' below).
        
        Simpler calls will be individual elements, such
        as ':12', which returns the integer 12.
        
        These are the possible prefixes; each item
        is followed by a \r\n, which is stripped
        by SocketLineReader:
        
            +<str>    simple string
            :<int>    integer
            $<n>    string of length <n>
            *<num>    start of array with <num> elements

        When the message to parse is the acknowledgment of
        a SUBSCRIBE or UNSUBSCRIBE command, this method
        will set() event self.unsubscribeAckEvent/self.unsubscribeAckEvent.

        :return: response string
        :rtype: string
        '''
        response = self._buffer.readline()
        if not response:
            raise ConnectionError(SERVER_CLOSED_CONNECTION_ERROR)

        byte, response = byte_to_chr(response[0]), response[1:]

        if byte not in ('-', '+', ':', '$', '*'):
            raise InvalidResponse("Protocol Error: %s, %s" %
                                  (str(byte), str(response)))

        # server returned an error
        if byte == '-':
            response = nativestr(response)
            error = self.parse_error(response)
            # if the error is a ConnectionError, raise immediately so the user
            # is notified
            if isinstance(error, ConnectionError):
                raise error
            # otherwise, we're dealing with a ResponseError that might belong
            # inside a pipeline response. the connection's read_response()
            # and/or the pipeline's execute() will raise this error if
            # necessary, so just return the exception instance here.
            return error
        # simple-string: response holds result:
        elif byte == '+':
            pass
        # int value
        elif byte == ':':
            response = long(response)
        # bulk response
        elif byte == '$':
            length = int(response)
            if length == -1:
                # Null string:
                return None
            response = self._buffer.read(length)
                        
        # multi-bulk response
        elif byte == '*':
            length = int(response)
            if length == -1:
                return None
            response = [self.read_response() for _ in xrange(length)]
        if isinstance(response, bytes) and self.encoding:
            response = response.decode(self.encoding)
        #***********
        #print('Response: %s' % byte + '|' + str(response))
        #***********
                
        return response

    def send_cluster_commands(self, stack, raise_on_error=True, allow_redirections=True):
        """
        Send a bunch of cluster commands to the redis cluster.

        `allow_redirections` If the pipeline should follow `ASK` & `MOVED` responses
        automatically. If set to false it will raise RedisClusterException.
        """
        if self.refresh_table_asap:
            self.connection_pool.nodes.initialize()
            self.refresh_table_asap = False

        ttl = self.RedisClusterRequestTTL
        response = {}
        attempt = xrange(0, len(stack)) if stack else []  # noqa
        ask_slots = {}

        while attempt and ttl > 0:
            ttl -= 1
            node_commands = {}
            nodes = {}

            # Keep this section so that we can determine what nodes to contact
            for i in attempt:
                c = stack[i]
                slot = self._determine_slot(*c[0])
                if slot in ask_slots:
                    node = ask_slots[slot]
                else:
                    node = self.connection_pool.nodes.slots[slot]

                self.connection_pool.nodes.set_node_name(node)
                node_name = node['name']
                nodes[node_name] = node
                node_commands.setdefault(node_name, {})
                node_commands[node_name][i] = c

            # Get one connection at a time from the pool and basiccly copy the logic from
            #  _execute_pipeline() below and do what a normal pipeline does.

            attempt = []
            if self.use_threads and len(node_commands) > 1:
                workers = dict()
                for node_name in node_commands:
                    node = nodes[node_name]
                    cmds = [node_commands[node_name][i] for i in sorted(node_commands[node_name].keys())]
                    with by_node_context(self.connection_pool, node) as connection:
                        workers[node_name] = Worker(self._execute_pipeline, connection, cmds, False)
                        workers[node_name].start()

                for node_name, worker in workers.items():
                    worker.join()
                    if worker.exception:
                        for i in sorted(node_commands[node_name].keys()):
                            response[i] = worker.exception
                    else:
                        for i, v in zip(sorted(node_commands[node_name].keys()), worker.value):
                            response[i] = v
                del workers
            else:
                for node_name in node_commands:
                    node = nodes[node_name]
                    cmds = [node_commands[node_name][i] for i in sorted(node_commands[node_name].keys())]
                    with by_node_context(self.connection_pool, node) as connection:
                        result = zip(
                            sorted(node_commands[node_name].keys()),
                            self._execute_pipeline(connection, cmds, False))
                        for i, v in result:
                            response[i] = v

            ask_slots = {}
            for i, v in response.items():
                if not isinstance(v, Exception):
                    continue

                if isinstance(v, ConnectionError):
                    ask_slots[self.connection_pool.nodes.keyslot(stack[i][0][1])] = random.choice(self.startup_nodes)
                    attempt.append(i)
                    if ttl < self.RedisClusterRequestTTL / 2:
                        time.sleep(0.1)
                    continue

                errv = StrictRedisCluster._exception_message(v)
                if errv is None:
                    continue

                if errv.startswith('CLUSTERDOWN'):
                    self.connection_pool.disconnect()
                    self.connection_pool.reset()
                    self.refresh_table_asap = True
                    raise ClusterDownException()

                redir = self.parse_redirection_exception_msg(errv)

                if not redir:
                    continue

                if redir['action'] == "MOVED":
                    self.refresh_table_asap = True
                    node = self.connection_pool.nodes.set_node(redir['host'], redir['port'], server_type='master')
                    self.connection_pool.nodes.slots[redir['slot']] = node
#.........这里部分代码省略.........

from redis._compat import xrange
from others.redis.cluster import rc
from others.redis.cluster import current_milli_time

s = current_milli_time()
for i in xrange(1000000):
    d = str(i)
    rc.set(d, d)
    rc.incrby(d, 1)

print current_milli_time() - s


# 1824127
# 1482678 32 conn

    def send_cluster_commands(self, stack, raise_on_error=True, allow_redirections=True):
        """
        Send a bunch of cluster commands to the redis cluster.

        `allow_redirections` If the pipeline should follow `ASK` & `MOVED` responses
        automatically. If set to false it will raise RedisClusterException.
        """
        if self.refresh_table_asap:
            self.connection_pool.nodes.initialize()
            self.refresh_table_asap = False

        ttl = self.RedisClusterRequestTTL

        # this is where we store all the responses to the pipeline execute.
        # the actual response is a list, not a dict, but because we may be getting
        # the responses for commands out of order or sometimes retrying them it is important
        # that we have a way to key the responses by the order they came in so that we can return
        # the responses as if we did them sequentially.
        response = {}

        # `attempt` corresponds to the sequence number of commands still left to process or retry.
        # initially this corresponds exactly to the number of commands we need to run, and if all goes
        # well, that's where it'll end. Everything will be attempted once, and we end up with an empty
        # array of commands left to process. But if we need to retry, `attempt` gets repopulated with
        # the sequence number of the command that is being retried.
        attempt = xrange(0, len(stack)) if stack else []  # noqa

        # there are different types of retries. redis cluster says if it responds with an ASK error,
        # you need to handle it differently than a moved error. And if we hit a connection error, we
        # don't really know what to do for that command so we pick a random other node in the cluster.
        ask_retry = {}
        conn_retry = {}

        # as long as we have commands left to attempt and we haven't overrun the max attempts, keep trying.
        while attempt and ttl > 0:

            # decrement our counter for number of attempts left before giving up.
            ttl -= 1

            # each time we go through this we need to separate out the commands by node.
            node_commands = {}

            # build a list of node objects based on node names we need to
            nodes = {}

            # as we move through each command that still needs to be processed,
            # we figure out the slot number that command maps to, then from the slot determine the node.
            slots = set()
            for i in attempt:
                c = stack[i]
                slot = self._determine_slot(*c[0])
                slots.add(slot)
                if self.explicit_transaction and len(slots) > 1:
                    raise WatchError("Cannot run explicit transaction against more than 1 hashslot, keys: {}."
                                     .format([str(c[0][1]) for c in stack]))

                # normally we just refer to our internal node -> slot table that tells us where a given
                # command should route to.
                # but if we are retrying, the cluster could have told us we were wrong or the node was down.
                # in that case, we have to try something that contradicts our rules.
                if i in ask_retry:
                    node = ask_retry[i]
                elif i in conn_retry:
                    node = conn_retry[i]
                else:
                    if self.refresh_table_asap:  # MOVED
                        node = self.connection_pool.get_master_node_by_slot(slot)
                    else:
                        node = self.connection_pool.get_node_by_slot(slot)

                # little hack to make sure the node name is populated. probably could clean this up.
                self.connection_pool.nodes.set_node_name(node)

                # now that we know the name of the node ( it's just a string in the form of host:port )
                # we can build a list of commands for each node.
                node_name = node['name']
                nodes[node_name] = node
                node_commands.setdefault(node_name, {})
                node_commands[node_name][i] = c

            # Get one connection at a time from the pool and basiccly copy the logic from
            #  _execute_pipeline() below and do what a normal pipeline does.

            # now that we've split out the commands by the node we plan to send them to,
            # we can reset the commands we'll attempt next time around back to nothing.
            # when we process the response, any commands that need to be retried because of
            # connection errors, MOVED errors, or ASKING errors will be dumped into here.
            # most of the time this array will just stay empty.
            attempt = []

            # only use threads when it makes sense performance-wise to do so.
            # if you are doing an ask_retry, explicitly disable it.
            # makes it easier to define the logic of how to do the ASKING command,
            # and in practice, the server will only be moving one slot at a time,
            # so there will only be one server that will be recieving these ASKING retries
            # anyway.
            if self.use_threads and not ask_retry and len(node_commands) > 1:

                # for each node we query, we need to have one worker.
                # that way all pipelined commands are issued in parallel.
#.........这里部分代码省略.........