Python MemcachedClientHelper.create_value方法代码示例

# 需要导入模块: from memcached.helper.data_helper import MemcachedClientHelper [as 别名]
# 或者: from memcached.helper.data_helper.MemcachedClientHelper import create_value [as 别名]
 def insert_key(serverInfo, bucket_name, count, size):
     client = MemcachedClientHelper.proxy_client(serverInfo, bucket_name)
     value = MemcachedClientHelper.create_value("*", size)
     for i in range(count * 1000):
         key = "key_" + str(i)
         flag = random.randint(1, 999)
         client.set(key, 0, flag, value)

# 需要导入模块: from memcached.helper.data_helper import MemcachedClientHelper [as 别名]
# 或者: from memcached.helper.data_helper.MemcachedClientHelper import create_value [as 别名]
 def test_append_wrong_cas(self):
     #monitor the memory usage , it should not go beyond
     #doing append 20,000 times ( each 5k) mem_used should not increase more than
     #10 percent
     #
     stats = self.onenodemc.stats()
     initial_mem_used = -1
     if "mem_used" in stats:
         initial_mem_used = int(stats["mem_used"])
         self.assertTrue(initial_mem_used > 0)
     key = str(uuid.uuid4())
     size = 5 * 1024
     value = MemcachedClientHelper.create_value("*", size)
     self.onenodemc.set(key, 0, 0, value)
     flags_v, cas_v, get_v = self.onenodemc.get(key)
     self.onenodemc.append(key, value, cas_v)
     iteration = 50000
     for i in range(0, iteration):
         try:
             self.onenodemc.append(key, value, random.randint(0, 1000))
         except:
             #ignoring the error here
             pass
     stats = self.onenodemc.stats()
     if "mem_used" in stats:
         delta = int(stats["mem_used"]) - initial_mem_used
         self.log.info("initial mem_used {0}, current mem_used {1}".format(initial_mem_used, stats["mem_used"]))
         self.log.info(delta)

# 需要导入模块: from memcached.helper.data_helper import MemcachedClientHelper [as 别名]
# 或者: from memcached.helper.data_helper.MemcachedClientHelper import create_value [as 别名]
    def _load_ops(self, ops=None, mutations=1, master=None, bucket=None):

        if master:
            self.rest = RestConnection(master)
        if bucket:
            self.client = VBucketAwareMemcached(self.rest, bucket)

        k=0
        payload = MemcachedClientHelper.create_value('*', self.value_size)

        while k < self.items:
            key = "{0}{1}".format(self.prefix, k)
            k += 1
            for i in range(mutations):
                if ops=='set':
                    #print 'set'
                    self.client.memcached(key).set(key, 0, 0,payload)
                elif ops=='add':
                    #print 'add'
                    self.client.memcached(key).add(key, 0, 0,payload)
                elif ops=='replace':
                    self.client.memcached(key).replace(key, 0, 0,payload)
                    #print 'Replace'
                elif ops=='delete':
                    #print 'delete'
                    self.client.memcached(key).delete(key)
                elif ops=='expiry':
                    #print 'expiry'
                    self.client.memcached(key).set(key, self.expire_time ,0, payload)
                elif ops=='touch':
                    #print 'touch'
                    self.client.memcached(key).touch(key, 10)

        self.log.info("Done with specified {0} ops".format(ops))

# 需要导入模块: from memcached.helper.data_helper import MemcachedClientHelper [as 别名]
# 或者: from memcached.helper.data_helper.MemcachedClientHelper import create_value [as 别名]
    def set_get_test(self, value_size, number_of_items):
        fixed_value = MemcachedClientHelper.create_value("S", value_size)
        specs = [
            ("default", 0),
            ("set-get-bucket-replica-1", 1),
            ("set-get-bucket-replica-2", 2),
            ("set-get-bucket-replica-3", 3),
        ]
        serverInfo = self.master
        rest = RestConnection(serverInfo)
        bucket_ram = int(rest.get_nodes_self().memoryQuota / 4)

        mcport = rest.get_nodes_self().memcached
        for name, replica in specs:
            rest.create_bucket(name, bucket_ram, "sasl", "password", replica, mcport)

        bucket_data = {}
        buckets = RestConnection(serverInfo).get_buckets()
        for bucket in buckets:
            bucket_data[bucket.name] = {}
            ready = BucketOperationHelper.wait_for_memcached(serverInfo, bucket.name)
            self.test.assertTrue(ready, "wait_for_memcached failed")

            client = MemcachedClientHelper.direct_client(serverInfo, bucket.name)
            inserted = []
            rejected = []
            while len(inserted) <= number_of_items and len(rejected) <= number_of_items:
                try:
                    key = str(uuid.uuid4())
                    client.set(key, 0, 0, fixed_value)
                    inserted.append(key)
                except mc_bin_client.MemcachedError:
                    pass

            retry = 0
            remaining_items = []
            remaining_items.extend(inserted)
            msg = "memcachedError : {0} - unable to get a pre-inserted key : {1}"
            while retry < 10 and len(remaining_items) > 0:
                verified_keys = []
                for key in remaining_items:
                    try:
                        flag, keyx, value = client.get(key=key)
                        if not value == fixed_value:
                            self.test.fail("value mismatch for key {0}".format(key))
                        verified_keys.append(key)
                    except mc_bin_client.MemcachedError as error:
                        self.log.error(msg.format(error.status, key))
                    retry += 1
                [remaining_items.remove(x) for x in verified_keys]

            print_count = 0
            for key in remaining_items:
                if print_count > 100:
                    break
                print_count += 1
                self.log.error("unable to verify key : {0}".format(key))
            if remaining_items:
                self.test.fail("unable to verify {0} keys".format(len(remaining_items)))

# 需要导入模块: from memcached.helper.data_helper import MemcachedClientHelper [as 别名]
# 或者: from memcached.helper.data_helper.MemcachedClientHelper import create_value [as 别名]
    def test_append_with_delete(self):
        #monitor the memory usage , it should not go beyond
        #doing append 20,000 times ( each 5k) mem_used should not increase more than
        #10 percent
        #
        if "iteration" in self.params:
            iteration = int(self.params["iteration"])
        else:
            iteration = 50000
        if "items" in self.params:
            items = int(self.params["items"])
        else:
            items = 10000
        if "append_size" in self.params:
            append_size = int(self.params["append_size"])
        else:
            append_size = 5 * 1024
        append_iteration_before_delete = 100
        keys = [str(uuid.uuid4()) for i in range(0, items)]
        size = 5 * 1024
        stats = self.onenodemc.stats()
        initial_mem_used = -1
        self.log.info("items : {0} , iteration : {1} ".format(items, iteration))
        if "mem_used" in stats:
            initial_mem_used = int(stats["mem_used"])
            self.assertTrue(initial_mem_used > 0)
        for i in range(0, iteration):
            for key in keys:
                self.onenodemc.set(key, 0, 0, os.urandom(size))
            try:
                for append_iteration in range(0, append_iteration_before_delete):
                    appened_value = MemcachedClientHelper.create_value("*", append_size)
                    for key in keys:
                        self.onenodemc.append(key, appened_value)
                        self.onenodemc.get(key)
            except:
                #ignoring the error here
                pass
            stats = None
            for t in range(0, 10):
                try:
                    stats = self.onenodemc.stats()
                    break
                except:
                    pass

            if stats and "mem_used" in stats:
                delta = int(stats["mem_used"]) - initial_mem_used
                #only print out if delta is more than 20% than it should be
                # delta ahould be #items * size + #items * append
                expected_delta = items * (size + append_iteration_before_delete * append_size * 1.0)
                msg = "initial mem_used {0}, current mem_used {1} , delta : {2} , expected delta : {3} , increase percentage {4}"
                self.log.info(
                    msg.format(initial_mem_used, stats["mem_used"], delta, expected_delta, delta / expected_delta))
                if delta > (1.2 * expected_delta):
                    self.fail("too much memory..")
                for key in keys:
                    self.onenodemc.delete(key)
            self.log.info   ("iteration #{0} completed".format(i))

# 需要导入模块: from memcached.helper.data_helper import MemcachedClientHelper [as 别名]
# 或者: from memcached.helper.data_helper.MemcachedClientHelper import create_value [as 别名]
 def _insert_data(self, client, howmany):
     prefix = str(uuid.uuid4())
     keys = ["{0}-{1}".format(prefix, i) for i in range(0, howmany)]
     value = MemcachedClientHelper.create_value("*", 1024)
     for key in keys:
         client.set(key, 0, 0, value)
     self.log.info("inserted {0} items".format(howmany))
     return keys

# 需要导入模块: from memcached.helper.data_helper import MemcachedClientHelper [as 别名]
# 或者: from memcached.helper.data_helper.MemcachedClientHelper import create_value [as 别名]
 def test_10k_items(self):
     keys = ["{0}-{1}".format(str(uuid.uuid4()), i) for i in range(0, 100)]
     value = MemcachedClientHelper.create_value("*", 1024)
     for k in keys:
         mc = self.smartclient
         vBucket = crc32.crc32_hash(k) & (mc.vbucket_count - 1)
         mc.set(k, 0, 0, value)
         mc.sync_persistence([{"key": k, "vbucket": vBucket}])

# 需要导入模块: from memcached.helper.data_helper import MemcachedClientHelper [as 别名]
# 或者: from memcached.helper.data_helper.MemcachedClientHelper import create_value [as 别名]
 def test_one_replica(self):
     self.common_setup(1)
     keys = ["{0}-{1}".format(str(uuid.uuid4()), i) for i in range(0, 100)]
     value = MemcachedClientHelper.create_value("*", 1024)
     for k in keys:
         vBucket = crc32.crc32_hash(k)
         mc = self.awareness.memcached(k)
         mc.set(k, 0, 0, value)
         mc.sync_replication(1, [{"key": k, "vbucket": vBucket}])
     for k in keys:
         mc = self.awareness.memcached(k)
         mc.get(k)

# 需要导入模块: from memcached.helper.data_helper import MemcachedClientHelper [as 别名]
# 或者: from memcached.helper.data_helper.MemcachedClientHelper import create_value [as 别名]
 def test_prepend_till_20_mb(self):
     initial_value = "12345678"
     key = str(uuid.uuid4())
     self.keys_cleanup.append(key)
     self.onenodemc.set(key, 0, 0, initial_value)
     #for 20 * 1024 times append 1024 chars each time
     value = MemcachedClientHelper.create_value("*", 1024 * 20)
     for i in range(0, (1024 - 1)):
         self.onenodemc.prepend(key, value)
     try:
         self.onenodemc.prepend(key, value)
         self.fail("memcached did not raise an error")
     except mc_bin_client.MemcachedError as err:
         self.assertEquals(err.status, 3)

# 需要导入模块: from memcached.helper.data_helper import MemcachedClientHelper [as 别名]
# 或者: from memcached.helper.data_helper.MemcachedClientHelper import create_value [as 别名]
 def test_10k_items_during_load(self):
     keys = ["{0}-{1}".format(str(uuid.uuid4()), i) for i in range(0, 100)]
     value = MemcachedClientHelper.create_value("*", 1024)
     prefix = str(uuid.uuid4())
     working_set_size = 10 * 1000
     self.load_thread = Thread(target=self._insert_data_till_stopped, args=("default", prefix, working_set_size))
     self.load_thread.start()
     for k in keys:
         mc = self.smartclient
         vBucket = crc32.crc32_hash(k) & (mc.vbucket_count - 1)
         mc.set(k, 0, 0, value)
         mc.sync_persistence([{"key": k, "vbucket": vBucket}])
     self.shutdown_load_data = True
     self.load_thread.join()

# 需要导入模块: from memcached.helper.data_helper import MemcachedClientHelper [as 别名]
# 或者: from memcached.helper.data_helper.MemcachedClientHelper import create_value [as 别名]
    def _unsupported_replicas(self, replica):
        self.common_setup(1)
        keys = ["{0}-{1}".format(str(uuid.uuid4()), i) for i in range(0, 100)]
        value = MemcachedClientHelper.create_value("*", 102400)
        for k in keys:
            vBucket = crc32.crc32_hash(k)
            mc = self.awareness.memcached(k)
            mc.set(k, 0, 0, value)
            mc.get(k)
            try:
                mc.sync_replication(replica, [{"key": k, "vbucket": vBucket}])
                msg = "server did not raise an error when running sync_replication with {0} replicas"
                self.fail(msg.format(replica))
            except MemcachedError as error:
                self.log.info("error {0} {1} as expected".format(error.status, error.msg))

        for k in keys:
            mc = self.awareness.memcached(k)
            mc.get(k)

# 需要导入模块: from memcached.helper.data_helper import MemcachedClientHelper [as 别名]
# 或者: from memcached.helper.data_helper.MemcachedClientHelper import create_value [as 别名]
 def touch_test(self):
     timeout = 900  # 15 minutes
     stats_all_buckets = {}
     payload = MemcachedClientHelper.create_value('*', self.value_size)
     mc = MemcachedClientHelper.proxy_client(self.servers[0], "default")
     prefix = "test_"
     self.num_items = self.input.param("items", 10000)
     k = 0
     while k < 100:
         key = "{0}{1}".format(prefix, k)
         mc.set(key, 0, 0, payload)
         k += 1
     active_resident_threshold = 30
     threshold_reached = False
     end_time = time.time() + float(timeout)
     while not threshold_reached and time.time() < end_time:
         if int(mc.stats()["vb_active_perc_mem_resident"]) >= active_resident_threshold:
             self.log.info("vb_active_perc_mem_resident_ratio reached at %s " % (mc.stats()["vb_active_perc_mem_resident"]))
             items = self.num_items
             self.num_items += self.input.param("items", 40000)
             random_key = self.key_generator()
             generate_load = BlobGenerator(random_key, '%s-' % random_key, self.value_size, end=self.num_items)
             self._load_all_buckets(self.servers[0], generate_load, "create", 0, True, batch_size=40000, pause_secs=3)
         else:
             threshold_reached = True
             self.log.info("DGM {0} state achieved!!!!".format(active_resident_threshold))
     if time.time() > end_time and int(mc.stats()["vb_active_perc_mem_resident"]) >= active_resident_threshold:
         raise Exception("failed to load items into bucket")
     """ at active resident ratio above, the first 100 keys
         insert into bucket will be non resident.  Then do touch command to test it """
     self.log.info("Run touch command to test items which are in non resident.")
     k = 0
     while k < 100:
         key = "{0}{1}".format(prefix, k)
         try:
             mc.touch(key, 0)
             k += 1
         except Exception as ex:
             raise Exception(ex)

# 需要导入模块: from memcached.helper.data_helper import MemcachedClientHelper [as 别名]
# 或者: from memcached.helper.data_helper.MemcachedClientHelper import create_value [as 别名]
 def test_not_your_vbucket(self):
     self.common_setup(1)
     keys = ["{0}-{1}".format(str(uuid.uuid4()), i) for i in range(0, 100)]
     value = MemcachedClientHelper.create_value("*", 1024)
     for k in keys:
         vBucket = crc32.crc32_hash(k)
         mc = self.awareness.memcached(k)
         mc.set(k, 0, 0, value)
         not_your_vbucket_mc = self.awareness.not_my_vbucket_memcached(k)
         try:
             count = 0
             expected_error = 0
             while count < 100:
                 a, b, response = not_your_vbucket_mc.sync_replication([{"key": k, "vbucket": vBucket}], 1)
                 count += 1
                 self.log.info("response : {0}".format(response))
                 if response and response[0]["event"] != "invalid key":
                     expected_error += 1
             if expected_error is not 100:
                 self.fail(msg="server did not raise an error when running sync_replication with invalid vbucket")
         except MemcachedError as error:
             self.log.error(error)

# 需要导入模块: from memcached.helper.data_helper import MemcachedClientHelper [as 别名]
# 或者: from memcached.helper.data_helper.MemcachedClientHelper import create_value [as 别名]
 params = input.test_params
 count = 0
 prefix = str(uuid.uuid4())[:6]
 count = 10 * 1000 * 1000 * 1000
 size = 512
 expiry = 0
 if "count" in params:
     count = int(params["count"])
     print count
 if "size" in params:
     size = int(params["size"])
 if "prefix" in params:
     prefix = params["prefix"]
 #if "expiry" in params:
 #    expiry = int(params["expiry"])
 payload = MemcachedClientHelper.create_value('*', size)
 rest = RestConnection(server)
 buckets = rest.get_buckets()
 for bucket in buckets:
     smart = VBucketAwareMemcached(rest, bucket.name)
     mc = MemcachedClientHelper.proxy_client(server, bucket.name)
     i = 0
     counter_10 = 0
     all_set = False
     while i < count:
         try:
             key = "{0}-{1}".format(prefix, i)
             #mc = smart.memcached(key)
             #do an expiry every 10 times
             if counter_10 >= 7:
                 if all_set == True:

# 需要导入模块: from memcached.helper.data_helper import MemcachedClientHelper [as 别名]
# 或者: from memcached.helper.data_helper.MemcachedClientHelper import create_value [as 别名]
def tap(server, queue):
    listen = TapListener(queue, server)
    listen.tap()


queue = Queue(maxsize=10000)

server = ServerInfo()
server.ip = "10.17.12.20"

bucket = {"name": "default", "port": 11220, "password": ""}
# vam = VBucketAwareMemcached(RestConnection(server), bucket)
# print vam.memcacheds
# print vam.vBucketMap
payload = MemcachedClientHelper.create_value("*", 10240)
keys = ["key_%d" % (i) for i in range(4000)]
# keys = ["key_%s_%d" % (str(uuid.uuid4()), i) for i in range(4)]
total_size = 0
# mc = MemcachedClientHelper.create_memcached_client("172.16.75.128","default",11210,"default")
mc = MemcachedClient("10.17.12.20", 11210)
# for key in keys:
#    vam.memcached(key).set(key, 1, 0, payload)
#    total_size += len(key) + len(payload) + 200
# time.sleep(10)
# for i in range(0,1023):
#    mc.set_vbucket_state(i, 'active')
new_thread = TapListener(queue, server)
new_thread.start()