Python hash.HashClient类代码示例

    def test_setup_client_without_pooling(self):
        with mock.patch('pymemcache.client.hash.Client') as internal_client:
            client = HashClient([], timeout=999, key_prefix='foo_bar_baz')
            client.add_server('127.0.0.1', '11211')

        assert internal_client.call_args[0][0] == ('127.0.0.1', '11211')
        kwargs = internal_client.call_args[1]
        assert kwargs['timeout'] == 999
        assert kwargs['key_prefix'] == 'foo_bar_baz'

    def test_no_servers_left_with_get_many(self):
        from pymemcache.client.hash import HashClient
        client = HashClient(
            [], use_pooling=True,
            ignore_exc=True,
            timeout=1, connect_timeout=1
        )

        result = client.get_many(['foo', 'bar'])
        assert result == {'foo': False, 'bar': False}

    def test_no_servers_left_with_commands(self):
        from pymemcache.client.hash import HashClient
        client = HashClient(
            [], use_pooling=True,
            ignore_exc=True,
            timeout=1, connect_timeout=1
        )

        result = client.get('foo')
        assert result is False

    def test_unavailable_servers_zero_retry_raise_exception(self):
        from pymemcache.client.hash import HashClient
        client = HashClient(
            [('example.com', 11211)], use_pooling=True,
            ignore_exc=False,
            retry_attempts=0, timeout=1, connect_timeout=1
        )

        with pytest.raises(socket.error):
            client.get('foo')

    def test_no_servers_left(self):
        from pymemcache.client.hash import HashClient
        client = HashClient(
            [], use_pooling=True,
            ignore_exc=True,
            timeout=1, connect_timeout=1
        )

        hashed_client = client._get_client('foo')
        assert hashed_client is None

    def test_no_servers_left_raise_exception(self):
        from pymemcache.client.hash import HashClient
        client = HashClient(
            [], use_pooling=True,
            ignore_exc=False,
            timeout=1, connect_timeout=1
        )

        with pytest.raises(MemcacheError) as e:
            client._get_client('foo')

        assert str(e.value) == 'All servers seem to be down right now'

    def make_client(self, *mock_socket_values, **kwargs):
        current_port = 11012
        client = HashClient([], **kwargs)
        ip = '127.0.0.1'

        for vals in mock_socket_values:
            s = '%s:%s' % (ip, current_port)
            c = self.make_client_pool(
                (ip, current_port),
                vals
            )
            client.clients[s] = c
            client.hasher.add_node(s)
            current_port += 1

        return client

def get_pymemcache_client(servers, timeout=None, **kwargs):
    from pymemcache.client.hash import HashClient

    from pymemcache.serde import (
        python_memcache_serializer, python_memcache_deserializer)

    kwargs.setdefault('serializer', python_memcache_serializer)
    kwargs.setdefault('deserializer', python_memcache_deserializer)

    if timeout:
        kwargs['timeout'] = timeout

    split = [s.split(':') for s in servers]
    _servers = [(host, int(port)) for host, port in split]
    client = HashClient(_servers, **kwargs)

    try:
        client.TooBig = ConnectionResetError
    except NameError:
        import socket
        client.TooBig = socket.error

    return client

    def _start(self):
        #self.client = pymemcache_client.PooledClient(
        from pymemcache.client.hash import HashClient
        self.client = HashClient(
            self.host,
            serializer=self._msgpack_serializer,
            deserializer=self._msgpack_deserializer,
            timeout=self.timeout,
            connect_timeout=self.timeout,
            max_pool_size=self.max_pool_size)

        # Run heartbeat here because pymemcache use a lazy connection
        # method and only connect once you do an operation.
        self.heartbeat()
        self._group_members = collections.defaultdict(set)
        self._executor.start()

  def setupMemcacheClient(self):
    """ Sets up the memcache client. """
    if os.path.exists(self.APPSCALE_MEMCACHE_FILE):
      memcache_file = open(self.APPSCALE_MEMCACHE_FILE, "r")
      all_ips = memcache_file.read().split("\n")
      memcache_file.close()
    else:
      all_ips = ['localhost']

    memcaches = [(ip, self.MEMCACHE_PORT) for ip in all_ips if ip]
    memcaches.sort()    
    self._memcache = HashClient(
      memcaches, serializer=serializer, deserializer=deserializer,
      connect_timeout=5, timeout=1, use_pooling=True)

    # The GAE API expects return values for all mutate operations.
    for client in six.itervalues(self._memcache.clients):
      client.default_noreply = False

 def _get_client(self, key):
     client = HashClient._get_client(self, key)
     client.default_noreply = False
     return client

class MemcacheService(apiproxy_stub.APIProxyStub):
  """Python only memcache service.

  This service keeps all data in any external servers running memcached.
  """
  # The memcached default port.
  MEMCACHE_PORT = 11211

  # An AppScale file which has a list of IPs running memcached.
  APPSCALE_MEMCACHE_FILE = "/etc/appscale/memcache_ips"

  def __init__(self, project_id, service_name='memcache'):
    """Initializer.

    Args:
      service_name: Service name expected for all calls.
    """
    super(MemcacheService, self).__init__(service_name)
    self._memcache = None
    self.setupMemcacheClient()
    self._methods = {MemcacheSetRequest.SET: self._memcache.set,
                     MemcacheSetRequest.ADD: self._memcache.add,
                     MemcacheSetRequest.REPLACE: self._memcache.replace,
                     MemcacheSetRequest.CAS: self._memcache.cas}
    self._project_id = project_id

  def setupMemcacheClient(self):
    """ Sets up the memcache client. """
    if os.path.exists(self.APPSCALE_MEMCACHE_FILE):
      memcache_file = open(self.APPSCALE_MEMCACHE_FILE, "r")
      all_ips = memcache_file.read().split("\n")
      memcache_file.close()
    else:
      all_ips = ['localhost']

    memcaches = [(ip, self.MEMCACHE_PORT) for ip in all_ips if ip]
    memcaches.sort()    
    self._memcache = HashClient(
      memcaches, serializer=serializer, deserializer=deserializer,
      connect_timeout=5, timeout=1, use_pooling=True)

    # The GAE API expects return values for all mutate operations.
    for client in six.itervalues(self._memcache.clients):
      client.default_noreply = False

  def _Dynamic_Get(self, request, response):
    """Implementation of gets for memcache.
     
    Args:
      request: A MemcacheGetRequest protocol buffer.
      response: A MemcacheGetResponse protocol buffer.
    """
    # Remove duplicate keys.
    original_keys = {
      encode_key(self._project_id, request.name_space(), key): key
      for key in request.key_list()}

    try:
      backend_response = self._memcache.get_many(
        original_keys.keys(), gets=request.for_cas())
    except MemcacheClientError as error:
      raise apiproxy_errors.ApplicationError(INVALID_VALUE, str(error))
    except TRANSIENT_ERRORS as error:
      raise apiproxy_errors.ApplicationError(
        UNSPECIFIED_ERROR, 'Transient memcache error: {}'.format(error))

    for encoded_key, value_tuple in six.iteritems(backend_response):
      item = response.add_item()
      item.set_key(original_keys[encoded_key])
      if request.for_cas():
        item.set_cas_id(int(value_tuple[1]))
        value_tuple = value_tuple[0]

      item.set_value(value_tuple[0])
      item.set_flags(value_tuple[1])

  def _Dynamic_Set(self, request, response):
    """Implementation of sets for memcache. 

    Args:
      request: A MemcacheSetRequest.
      response: A MemcacheSetResponse.
    """
    namespace = request.name_space()
    if any(item.set_policy() not in self._methods
           for item in request.item_list()):
      raise apiproxy_errors.ApplicationError(
        INVALID_VALUE, 'Unsupported set_policy')

    if not all(item.has_cas_id() for item in request.item_list()
               if item.set_policy() == MemcacheSetRequest.CAS):
      raise apiproxy_errors.ApplicationError(
        INVALID_VALUE, 'All CAS items must have a cas_id')

    for item in request.item_list():
      try:
        encoded_key = encode_key(self._project_id, namespace, item.key())
      except apiproxy_errors.ApplicationError:
        response.add_set_status(MemcacheSetResponse.ERROR)
        continue
#.........这里部分代码省略.........

class MemcachedDriver(coordination._RunWatchersMixin,
                      coordination.CoordinationDriver):
    """A `memcached`_ based driver.

    This driver users `memcached`_ concepts to provide the coordination driver
    semantics and required API(s). It **is** fully functional and implements
    all of the coordination driver API(s). It stores data into memcache
    using expiries and `msgpack`_ encoded values.

    General recommendations/usage considerations:

    - Memcache (without different backend technology) is a **cache** enough
      said.

    .. _memcached: http://memcached.org/
    .. _msgpack: http://msgpack.org/
    """

    CHARACTERISTICS = (
        coordination.Characteristics.DISTRIBUTED_ACROSS_THREADS,
        coordination.Characteristics.DISTRIBUTED_ACROSS_PROCESSES,
        coordination.Characteristics.DISTRIBUTED_ACROSS_HOSTS,
        coordination.Characteristics.CAUSAL,
    )
    """
    Tuple of :py:class:`~tooz.coordination.Characteristics` introspectable
    enum member(s) that can be used to interogate how this driver works.
    """

    #: Key prefix attached to groups (used in name-spacing keys)
    GROUP_PREFIX = b'_TOOZ_GROUP_'

    #: Key prefix attached to leaders of groups (used in name-spacing keys)
    GROUP_LEADER_PREFIX = b'_TOOZ_GROUP_LEADER_'

    #: Key prefix attached to members of groups (used in name-spacing keys)
    MEMBER_PREFIX = b'_TOOZ_MEMBER_'

    #: Key where all groups 'known' are stored.
    GROUP_LIST_KEY = b'_TOOZ_GROUP_LIST'

    #: Default socket/lock/member/leader timeout used when none is provided.
    DEFAULT_TIMEOUT = 30

    #: String used to keep a key/member alive (until it next expires).
    STILL_ALIVE = b"It's alive!"

    def __init__(self, member_id, parsed_url, options):
        super(MemcachedDriver, self).__init__()
        options = utils.collapse(options)
        self._options = options
        self._member_id = member_id
        self._joined_groups = set()
        self._executor = utils.ProxyExecutor.build("Memcached", options)
        # self.host = (parsed_url.hostname or "localhost",
        #              parsed_url.port or 11211)
        self.host = []
        for one_url in parsed_url:
            tmp = (one_url.hostname or "localhost",
                   one_url.port or 11211)
            self.host.append(tmp)
        default_timeout = options.get('timeout', self.DEFAULT_TIMEOUT)
        self.timeout = int(default_timeout)
        self.membership_timeout = int(options.get(
            'membership_timeout', default_timeout))
        self.lock_timeout = int(options.get(
            'lock_timeout', default_timeout))
        self.leader_timeout = int(options.get(
            'leader_timeout', default_timeout))
        max_pool_size = options.get('max_pool_size', None)
        if max_pool_size is not None:
            self.max_pool_size = int(max_pool_size)
        else:
            self.max_pool_size = None
        self._acquired_locks = []

    @staticmethod
    def _msgpack_serializer(key, value):
        if isinstance(value, six.binary_type):
            return value, 1
        return utils.dumps(value), 2

    @staticmethod
    def _msgpack_deserializer(key, value, flags):
        if flags == 1:
            return value
        if flags == 2:
            return utils.loads(value)
        raise coordination.SerializationError("Unknown serialization"
                                              " format '%s'" % flags)

    @_translate_failures
    def _start(self):
        #self.client = pymemcache_client.PooledClient(
        from pymemcache.client.hash import HashClient
        self.client = HashClient(
            self.host,
            serializer=self._msgpack_serializer,
            deserializer=self._msgpack_deserializer,
            timeout=self.timeout,
#.........这里部分代码省略.........

 def __init__(self, couchbase_uri, memcached_hosts, primary=PRIMARY_COUCHBASE):
     """
     :param couchbase_uri: Connection string for Couchbase
     :param memcached_hosts: List of Memcached nodes
     :param primary: Determines which datastore is authoritative.
         This affects how get operations are performed and which datastore
         is used for CAS operations.
             PRIMARY_COUCHBASE: Couchbase is authoritative
             PRIMARY_MEMCACHED: Memcached is authoritative
         By default, Couchbase is the primary store
     :return:
     """
     self.cb = CbBucket(couchbase_uri)
     self.mc = McClient(memcached_hosts)
     self._primary = primary

class CouchbaseMemcacheMirror(object):
    def __init__(self, couchbase_uri, memcached_hosts, primary=PRIMARY_COUCHBASE):
        """
        :param couchbase_uri: Connection string for Couchbase
        :param memcached_hosts: List of Memcached nodes
        :param primary: Determines which datastore is authoritative.
            This affects how get operations are performed and which datastore
            is used for CAS operations.
                PRIMARY_COUCHBASE: Couchbase is authoritative
                PRIMARY_MEMCACHED: Memcached is authoritative
            By default, Couchbase is the primary store
        :return:
        """
        self.cb = CbBucket(couchbase_uri)
        self.mc = McClient(memcached_hosts)
        self._primary = primary

    @property
    def primary(self):
        return self._primary

    def _cb_get(self, key):
        try:
            return self.cb.get(key).value
        except NotFoundError:
            return None

    def get(self, key, try_alternate=True):
        """
        Gets a document
        :param key: The key to retrieve
        :param try_alternate: Whether to try the secondary data source if the
            item is not found in the primary.
        :return: The value as a Python object
        """
        if self._primary == PRIMARY_COUCHBASE:
            order = [self._cb_get, self.mc.get]
        else:
            order = [self.mc.get, self._cb_get]

        for meth in order:
            ret = meth(key)
            if ret or not try_alternate:
                return ret

        return None

    def _cb_mget(self, keys):
        """
        Internal method to execute a Couchbase multi-get
        :param keys: The keys to retrieve
        :return: A tuple of {found_key:found_value, ...}, [missing_key1,...]
        """
        try:
            ok_rvs = self.cb.get_multi(keys)
            bad_rvs = {}
        except NotFoundError as e:
            ok_rvs, bad_rvs = e.split_results()

        ok_dict = {k: (v.value, v.cas) for k, v in ok_rvs}
        return ok_dict, bad_rvs.keys()

    def get_multi(self, keys, try_alternate=True):
        """
        Gets multiple items from the server
        :param keys: The keys to fetch as an iterable
        :param try_alternate: Whether to fetch missing items from alternate store
        :return: A dictionary of key:value. Only contains keys which exist and have values
        """
        if self._primary == PRIMARY_COUCHBASE:
            ok, err = self._cb_get(keys)
            if err and try_alternate:
                ok.update(self.mc.get_many(err))
            return ok
        else:
            ok = self.mc.get_many(keys)
            if len(ok) < len(keys) and try_alternate:
                keys_err = set(keys) - set(ok)
                ok.update(self._cb_mget(list(keys_err))[0])
            return ok

    def gets(self, key):
        """
        Get an item with its CAS. The item will always be fetched from the primary
        data store.

        :param key: the key to get
        :return: the value of the key, or None if no such value
        """
        if self._primary == PRIMARY_COUCHBASE:
            try:
                rv = self.cb.get(key)
                return key, rv.cas
            except NotFoundError:
                return None, None
        else:
            return self.mc.gets(key)

    def gets_multi(self, keys):
        if self._primary == PRIMARY_COUCHBASE:
#.........这里部分代码省略.........