Python RestConnection.all_docs方法代码示例

# 需要导入模块: from membase.api.rest_client import RestConnection [as 别名]
# 或者: from membase.api.rest_client.RestConnection import all_docs [as 别名]
    def verify_es_num_docs(self, src_server, dest_server, kv_store=1, retry=10, verification_count=10000):
        cb_rest = RestConnection(src_server)
        es_rest = RestConnection(dest_server)
        buckets = self.xd_ref._get_cluster_buckets(src_server)
        wait = 20
        for bucket in buckets:
            all_cb_docs = cb_rest.all_docs(bucket.name)
            cb_valid = [str(row["id"]) for row in all_cb_docs["rows"]]
            cb_num_items = cb_rest.get_bucket_stats(bucket.name)["curr_items"]
            es_num_items = es_rest.get_bucket(bucket.name).stats.itemCount
            _retry = retry
            while _retry > 0 and cb_num_items != es_num_items:
                self._log.info(
                    "elasticsearch items %s, expected: %s....retry after %s seconds"
                    % (es_num_items, cb_num_items, wait)
                )
                time.sleep(wait)
                last_es_items = es_num_items
                es_num_items = es_rest.get_bucket(bucket.name).stats.itemCount
                if es_num_items == last_es_items:
                    _retry = _retry - 1
                    # if index doesn't change reduce retry count
                elif es_num_items <= last_es_items:
                    self._log.info("%s items removed from index " % (es_num_items - last_es_items))
                    _retry = retry
                elif es_num_items >= last_es_items:
                    self._log.info("%s items added to index" % (es_num_items - last_es_items))
                    _retry = retry

            if es_num_items != cb_num_items:
                self.xd_ref.fail(
                    "Error: Couchbase has %s docs, ElasticSearch has %s docs " % (cb_num_items, es_num_items)
                )

            # query for all es keys
            es_valid = es_rest.all_docs(keys_only=True, indices=[bucket.name], size=cb_num_items)

            if len(es_valid) != cb_num_items:
                self._log.info(
                    "WARNING: Couchbase has %s docs, ElasticSearch all_docs returned %s docs "
                    % (cb_num_items, len(es_valid))
                )
            for _id in cb_valid[:verification_count]:  # match at most 10k keys
                id_found = _id in es_valid

                if id_found == False:
                    # document missing from all_docs query do manual term search
                    doc = es_rest.search_all_nodes(_id, indices=[bucket.name])
                    if doc is None:
                        self.xd_ref.fail("Document %s Missing from ES Index (%s)" % (_id, bucket.name))

            self._log.info(
                "Verified couchbase bucket (%s) replicated (%s) docs to elasticSearch with matching keys"
                % (bucket.name, cb_num_items)
            )

# 需要导入模块: from membase.api.rest_client import RestConnection [as 别名]
# 或者: from membase.api.rest_client.RestConnection import all_docs [as 别名]
    def _verify_es_values(self, src_server, dest_server, kv_store=1, verification_count=10000):
        cb_rest = RestConnection(src_server)
        es_rest = RestConnection(dest_server)
        buckets = self.xd_ref._get_cluster_buckets(src_server)
        for bucket in buckets:
            mc = VBucketAwareMemcached(cb_rest, bucket)
            es_valid = es_rest.all_docs(indices=[bucket.name], size=verification_count)

            # compare values of es documents to documents in couchbase
            for row in es_valid[:verification_count]:
                key = str(row["meta"]["id"])

                try:
                    _, _, doc = mc.get(key)
                    val_src = str(json.loads(doc)["site_name"])
                    val_dest = str(row["doc"]["site_name"])
                    if val_src != val_dest:
                        self.xd_ref.fail(
                            "Document %s has unexpected value (%s) expected (%s)" % (key, val_src, val_dest)
                        )
                except MemcachedError as e:
                    self.xd_ref.fail("Error during verification.  Index contains invalid key: %s" % key)

            self._log.info(
                "Verified doc values in couchbase bucket (%s) match values in elastic search" % (bucket.name)
            )

# 需要导入模块: from membase.api.rest_client import RestConnection [as 别名]
# 或者: from membase.api.rest_client.RestConnection import all_docs [as 别名]
    def _verify_es_revIds(self, src_server, dest_server, kv_store=1, verification_count=10000):
        cb_rest = RestConnection(src_server)
        es_rest = RestConnection(dest_server)
        buckets = self.xd_ref._get_cluster_buckets(src_server)
        for bucket in buckets:

            # retrieve all docs from couchbase and es
            # then represent doc lists from couchbase
            # and elastic search in following format
            # [  (id, rev), (id, rev) ... ]
            all_cb_docs = cb_rest.all_docs(bucket.name)
            cb_id_rev_list = self.get_cb_id_rev_list(all_cb_docs)

            es_valid = es_rest.all_docs(indices=[bucket.name], size=len(cb_id_rev_list))
            es_id_rev_list = self.get_es_id_rev_list(es_valid)

            # verify each (id, rev) pair returned from couchbase exists in elastic search
            for cb_id_rev_pair in cb_id_rev_list:

                try:
                    # lookup es document with matching _id and revid
                    # if no exception thrown then doc was properly indexed
                    es_list_pos = es_id_rev_list.index(cb_id_rev_pair)
                except ValueError:

                    # attempt manual lookup by search term
                    es_doc = es_rest.search_all_nodes(cb_id_rev_pair[0], indices=[bucket.name])

                    if es_doc is None:
                        self.xd_ref.fail(
                            "Error during verification:  %s does not exist in ES index %s"
                            % (cb_id_rev_pair, bucket.name)
                        )

                    # compare
                    es_id_rev_pair = (es_doc["_source"]["meta"]["id"], es_doc["_source"]["meta"]["rev"])
                    if cb_id_rev_pair != es_id_rev_pair:
                        self.xd_ref.fail(
                            "ES document %s Missmatch Couchbase doc (%s). bucket (%s)"
                            % (es_id_rev_pair, cb_id_rev_pair, bucket.name)
                        )

            self._log.info(
                "Verified doc rev-ids in couchbase bucket (%s) match meta rev-ids elastic search" % (bucket.name)
            )

# 需要导入模块: from membase.api.rest_client import RestConnection [as 别名]
# 或者: from membase.api.rest_client.RestConnection import all_docs [as 别名]
    def _verify_keys(self, doc_names, prefix):
        master = self.servers[0]
        rest = RestConnection(master)
        bucket = "default"

        results = rest.all_docs(bucket)
        result_keys = ViewBaseTests._get_keys(self, results)

        # result_keys also contains the design document, but this doesn't
        # matter if we compare like this
        if set(doc_names) - set(result_keys):
            self.fail("returned doc names have different values than expected")

# 需要导入模块: from membase.api.rest_client import RestConnection [as 别名]
# 或者: from membase.api.rest_client.RestConnection import all_docs [as 别名]
    def test_all_docs_startkey_endkey_validation(self):
        """Regression test for MB-6591

        This tests makes sure that the validation of startkey/endkey
        works with _all_docs which uses raw collation. Return results
        when startkey is smaller than endkey. When endkey is smaller than
        the startkey and exception should be raised. With raw collation
        "Foo" < "foo", with Unicode collation "foo" < "Foo".
        """
        bucket = "default"
        master = self.servers[0]
        rest = RestConnection(master)
        ViewBaseTests._setup_cluster(self)

        startkey = '"Foo"'
        endkey = '"foo"'

        params = {"startkey": startkey, "endkey": endkey}
        results = rest.all_docs(bucket, params)
        self.assertTrue('rows' in results, "Results were returned")

        # Flip startkey and endkey
        params = {"startkey": endkey, "endkey": startkey}
        self.assertRaises(Exception, rest.all_docs, bucket, params)