Python SpatialHelper.insert_docs方法代码示例

# 需要导入模块: from membase.helper.spatial_helper import SpatialHelper [as 别名]
# 或者: from membase.helper.spatial_helper.SpatialHelper import insert_docs [as 别名]
class SpatialInfoTests(unittest.TestCase):
    def setUp(self):
        self.log = logger.Logger.get_logger()
        self.helper = SpatialHelper(self, "default")
        self.helper.setup_cluster()


    def tearDown(self):
        self.helper.cleanup_cluster()


    def test_spatial_info(self):
        self.log.info(
            "description : test info for spatial indexes")
        rest = self.helper.rest
        prefix = str(uuid.uuid4())[:7]
        design_name = "dev_test_spatial_info"

        self.helper.create_index_fun(design_name, prefix)

        # Fill the database and add an index
        self.helper.insert_docs(2000, prefix)
        self.helper.get_results(design_name)
        status, info = self.helper.info(design_name)
        disk_size = info["spatial_index"]["disk_size"]

        self.assertTrue(disk_size > 0)
        self.assertEqual(info["name"], design_name)

        num_vbuckets = len(rest.get_vbuckets(self.helper.bucket))
        self.assertEqual(len(info["spatial_index"]["update_seq"]),
                         num_vbuckets)
        self.assertEqual(len(info["spatial_index"]["purge_seq"]),
                         num_vbuckets)
        self.assertFalse(info["spatial_index"]["updater_running"])
        self.assertFalse(info["spatial_index"]["waiting_clients"] > 0)
        self.assertFalse(info["spatial_index"]["compact_running"])

        # Insert a lot new documents, and return after starting to
        # build up (not waiting until it's done) the index to test
        # if the updater fields are set correctly
        self.helper.insert_docs(50000, prefix)
        self.helper.get_results(design_name,
                                extra_params={"stale": "update_after"})
        # Somehow stale=update_after doesn't really return immediately,
        # thus commenting this assertion out. There's no real reason
        # to investigate, as the indexing changes heavily in the moment
        # anyway
        #self.assertTrue(info["spatial_index"]["updater_running"])
        #self.assertTrue(info["spatial_index"]["waiting_commit"])
        #self.assertTrue(info["spatial_index"]["waiting_clients"] > 0)
        self.assertFalse(info["spatial_index"]["compact_running"])

        # Request the index again, to make sure it is fully updated
        self.helper.get_results(design_name)
        status, info = self.helper.info(design_name)
        self.assertFalse(info["spatial_index"]["updater_running"])
        self.assertFalse(info["spatial_index"]["waiting_clients"] > 0)
        self.assertFalse(info["spatial_index"]["compact_running"])
        self.assertTrue(info["spatial_index"]["disk_size"] > disk_size)

# 需要导入模块: from membase.helper.spatial_helper import SpatialHelper [as 别名]
# 或者: from membase.helper.spatial_helper.SpatialHelper import insert_docs [as 别名]
class SpatialCompactionTests(BaseTestCase):
    def setUp(self):
        super(SpatialCompactionTests, self).setUp()
        self.start_cluster = self.input.param('start-cluster', len(self.servers))
        self.servers_in = self.input.param('servers_in', 0)
        self.servers_out = self.input.param('servers_out', 0)
        self.bucket_name = "default"
        if self.standard_buckets:
            self.bucket_name = "standard_bucket0"
        if self.sasl_buckets:
            self.bucket_name = "bucket0"
        self.helper = SpatialHelper(self, self.bucket_name)
        if self.start_cluster > 1:
            rebalance = self.cluster.async_rebalance(self.servers[:1],
                                                     self.servers[1:start_cluster], [])
            rebalance.result()

    def tearDown(self):
        super(SpatialCompactionTests, self).tearDown()


    def test_spatial_compaction(self):
        self.log.info(
            "description : test manual compaction for spatial indexes")
        prefix = str(uuid.uuid4())[:7]
        design_name = "dev_test_spatial_compaction"

        self.helper.create_index_fun(design_name, prefix)

        # Insert (resp. update, as they have the same prefix) and query
        # the spatial index several time so that the compaction makes sense
        for i in range(0, 8):
            self.helper.insert_docs(2000, prefix)
            self.helper.get_results(design_name)

        # Get the index size prior to compaction
        status, info = self.helper.info(design_name)
        disk_size = info["spatial_index"]["disk_size"]

        if self.servers_in or self.servers_out:
            servs_in = servs_out = []
            if self.servers_in:
                servs_in = self.servers[self.start_cluster:self.servers_in + 1]
            if self.servers_out:
                servs_out = self.servers[-self.servers_out:]
            rebalance = self.cluster.async_rebalance(self.servers, servs_in, servs_out)

        # Do the compaction
        self.helper.compact(design_name)

        # Check if the index size got smaller
        status, info = self.helper.info(design_name)
        self.assertTrue(info["spatial_index"]["disk_size"] < disk_size,
                        "The file size ({0}) isn't smaller than the "
                        "pre compaction size ({1})."
                        .format(info["spatial_index"]["disk_size"],
                                disk_size))
        if self.servers_in or self.servers_out:
            rebalance.result()

# 需要导入模块: from membase.helper.spatial_helper import SpatialHelper [as 别名]
# 或者: from membase.helper.spatial_helper.SpatialHelper import insert_docs [as 别名]
class SpatialCompactionTests(unittest.TestCase):
    def setUp(self):
        self.log = logger.Logger.get_logger()
        self.helper = SpatialHelper(self, "default")
        self.helper.setup_cluster()


    def tearDown(self):
        self.helper.cleanup_cluster()


    def test_spatial_compaction(self):
        self.log.info(
            "description : test manual compaction for spatial indexes")
        rest = self.helper.rest
        prefix = str(uuid.uuid4())[:7]
        design_name = "dev_test_spatial_compaction"

        self.helper.create_index_fun(design_name, prefix)

        # Insert (resp. update, as they have the same prefix) and query
        # the spatial index several time so that the compaction makes sense
        for i in range(0, 8):
            doc_names = self.helper.insert_docs(2000, prefix)
            self.helper.get_results(design_name)

        # Get the index size prior to compaction
        status, info = self.helper.info(design_name)
        disk_size = info["spatial_index"]["disk_size"]

        # Do the compaction
        self.helper.compact(design_name)

        # Check if the index size got smaller
        status, info = self.helper.info(design_name)
        self.assertTrue(info["spatial_index"]["disk_size"] < disk_size,
                        "The file size ({0}) isn't smaller than the "
                        "pre compaction size ({1})."
                        .format(info["spatial_index"]["disk_size"],
                                disk_size))

# 需要导入模块: from membase.helper.spatial_helper import SpatialHelper [as 别名]
# 或者: from membase.helper.spatial_helper.SpatialHelper import insert_docs [as 别名]
class SpatialViewTests(unittest.TestCase):
    def setUp(self):
        self.log = logger.Logger.get_logger()
        self.helper = SpatialHelper(self, "default")
        self.helper.setup_cluster()


    def tearDown(self):
        self.helper.cleanup_cluster()


    def test_create_x_design_docs(self):
        num_design_docs = self.helper.input.param("num-design-docs", 5)
        self.log.info("description : create {0} spatial views without "
                      "running any spatial view query".format(num_design_docs))
        prefix = str(uuid.uuid4())

        fun = "function (doc) {emit(doc.geometry, doc);}"
        self._insert_x_design_docs(num_design_docs, prefix, fun)


    def test_update_x_design_docs(self):
        num_design_docs = self.helper.input.param("num-design-docs", 5)
        self.log.info("description : update {0} spatial views without "
                      "running any spatial view query".format(num_design_docs))
        prefix = str(uuid.uuid4())

        fun = "function (doc) {emit(doc.geometry, doc);}"
        self._insert_x_design_docs(num_design_docs, prefix, fun)

        # Update the design docs with a different function
        fun = "function (doc) {emit(doc.geometry, null);}"
        self._insert_x_design_docs(num_design_docs, prefix, fun)


    def _insert_x_design_docs(self, num_design_docs, prefix, fun):
        rest = self.helper.rest
        bucket = self.helper.bucket
        name = "dev_test_multiple_design_docs"

        for i in range(0, num_design_docs):
            design_name = "{0}-{1}-{2}".format(name, i, prefix)
            self.helper.create_index_fun(design_name, prefix, fun)

            # Verify that the function was really stored
            response, meta = rest.get_spatial(bucket, design_name)
            self.assertTrue(response)
            self.assertEquals(meta["id"],
                              "_design/{0}".format(design_name))
            self.assertEquals(
                response["spatial"][design_name].encode("ascii",
                                                                "ignore"),
                fun)


    def test_insert_x_docs(self):
        num_docs = self.helper.input.param("num-docs", 100)
        self.log.info("description : create a spatial view on {0} documents"\
                          .format(num_docs))
        design_name = "dev_test_insert_{0}_docs".format(num_docs)
        self._insert_x_docs_and_query(num_docs, design_name)


    # Does verify the full docs and not only the keys
    def test_insert_x_docs_full_verification(self):
        num_docs = self.helper.input.param("num-docs", 100)
        self.log.info("description : create a spatial view with {0} docs"
                      " and verify the full documents".format(num_docs))
        design_name = "dev_test_insert_{0}_docs_full_verification"\
            .format(num_docs)
        prefix = str(uuid.uuid4())[:7]

        self.helper.create_index_fun(design_name, prefix)
        inserted_docs = self.helper.insert_docs(num_docs, prefix,
                                                return_docs=True)
        self.helper.query_index_for_verification(design_name, inserted_docs,
                                                 full_docs=True)


    def test_insert_x_delete_y_docs(self):
        num_docs = self.helper.input.param("num-docs", 15000)
        num_deleted_docs = self.helper.input.param("num-deleted-docs", 10000)
        self.log.info("description : create spatial view with {0} docs "
                      " and delete {1} docs".format(num_docs,
                                                    num_deleted_docs))
        design_name = "dev_test_insert_{0}_delete_{1}_docs"\
            .format(num_docs, num_deleted_docs)
        prefix = str(uuid.uuid4())[:7]

        inserted_keys = self._setup_index(design_name, num_docs, prefix)

        # Delete documents and verify that the documents got deleted
        deleted_keys = self.helper.delete_docs(num_deleted_docs, prefix)
        num_expected = num_docs - len(deleted_keys)
        results = self.helper.get_results(design_name, 2 * num_docs,
                                          num_expected=num_expected)
        result_keys = self.helper.get_keys(results)
        self.assertEqual(len(result_keys), num_expected)
        self.helper.verify_result(inserted_keys, deleted_keys + result_keys)

#.........这里部分代码省略.........

# 需要导入模块: from membase.helper.spatial_helper import SpatialHelper [as 别名]
# 或者: from membase.helper.spatial_helper.SpatialHelper import insert_docs [as 别名]
class SpatialViewsTests(BaseTestCase):

    def setUp(self):
        super(SpatialViewsTests, self).setUp()
        self.skip_rebalance = self.input.param("skip_rebalance", False)
        self.use_dev_views = self.input.param("use-dev-views", False)
        self.default_map = "function (doc) {emit(doc.geometry, doc.age);}"
        self.default_ddoc_name = self.input.param("default_ddoc_name", "test-ddoc")
        self.default_view_name = self.input.param("default_view_name", "test-view")
        self.bucket_name = "default"
        if self.standard_buckets:
            self.bucket_name = "standard_bucket0"
        if self.sasl_buckets:
            self.bucket_name = "bucket0"
        self.helper = SpatialHelper(self, self.bucket_name)
        if not self.skip_rebalance:
            self.cluster.rebalance(self.servers[:], self.servers[1:], [])
        #load some items to verify
        self.docs = self.helper.insert_docs(self.num_items, 'spatial-doc',
                                            wait_for_persistence=True,
                                            return_docs=True)

    def tearDown(self):
        super(SpatialViewsTests, self).tearDown()

    def test_add_spatial_views(self):
        num_ddoc = self.input.param('num-ddoc', 1)
        views_per_ddoc = self.input.param('views-per-ddoc', 1)
        non_spatial_views_per_ddoc = self.input.param('non-spatial-views-per-ddoc', 0)
        ddocs =  self.make_ddocs(num_ddoc, views_per_ddoc, non_spatial_views_per_ddoc)
        self.create_ddocs(ddocs)

    def test_add_spatial_views_case_sensative(self):
        ddoc = DesignDocument(self.default_ddoc_name, [], spatial_views=[
                                  View(self.default_view_name, self.default_map,
                                       dev_view=self.use_dev_views, is_spatial=True),
                                  View(self.default_view_name.upper(), self.default_map,
                                       dev_view=self.use_dev_views, is_spatial=True)])
        self.create_ddocs([ddoc])

    def make_ddocs(self, ddocs_num, views_per_ddoc, non_spatial_views_per_ddoc):
        ddocs = []
        for i in xrange(ddocs_num):
            views = []
            for k in xrange(views_per_ddoc):
                views.append(View(self.default_view_name + str(k), self.default_map,
                                  dev_view=self.use_dev_views, is_spatial=True))
            non_spatial_views = []
            if non_spatial_views_per_ddoc:
                for k in xrange(non_spatial_views_per_ddoc):
                    views.append(View(self.default_view_name + str(k), 'function (doc) { emit(null, doc);}',
                                      dev_view=self.use_dev_views))
            ddocs.append(DesignDocument(self.default_ddoc_name + str(i), non_spatial_views, spatial_views=views))
        return ddocs

    def create_ddocs(self, ddocs):
        for ddoc in ddocs:
            if not (ddoc.views or ddoc.spatial_views):
                self.cluster.create_view(self.master, ddoc.name, [], bucket=self.bucket_name)
            for view in ddoc.views:
                self.cluster.create_view(self.master, ddoc.name, view, bucket=self.bucket_name)
            for view in ddoc.spatial_views:
                self.cluster.create_view(self.master, ddoc.name, view, bucket=self.bucket_name)

# 需要导入模块: from membase.helper.spatial_helper import SpatialHelper [as 别名]
# 或者: from membase.helper.spatial_helper.SpatialHelper import insert_docs [as 别名]
class SpatialViewQueriesTests(BaseTestCase):

    def setUp(self):
        super(SpatialViewQueriesTests, self).setUp()
        self.thread_crashed = Event()
        self.thread_stopped = Event()
        self.skip_rebalance = self.input.param("skip_rebalance", False)
        self.use_dev_views = self.input.param("use-dev-views", False)
        self.all_view_one_ddoc = self.input.param("all-view-one-ddoc", False)
        self.default_ddoc_name = "test-ddoc-query"
        self.default_view_name = "test-view-query"
        self.params = self.get_query_params()
        self.bucket_name = "default"
        if self.standard_buckets:
            self.bucket_name = "standard_bucket0"
        if self.sasl_buckets:
            self.bucket_name = "bucket0"
        self.helper = SpatialHelper(self, self.bucket_name)
        if not self.skip_rebalance:
            self.cluster.rebalance(self.servers[:], self.servers[1:], [])
        #load some items to verify
        self.docs = self.helper.insert_docs(self.num_items, 'spatial-doc',
                                            return_docs=True)
        self.ddocs = self.helper.create_default_views(
                                        is_one_ddoc=self.all_view_one_ddoc)

    def tearDown(self):
        super(SpatialViewQueriesTests, self).tearDown()

    def test_spatial_view_queries(self):
        error = self.input.param('error', None)
        try:
            self.query_and_verify_result(self.docs, self.params)
        except Exception as ex:
            if error and str(ex).find(error) != -1:
               self.log.info("Error caught as expected %s" % error)
               return
            else:
               self.fail("Unexpected error appeared during run %s" % ex)
        if error:
            self.fail("Expected error '%s' didn't appear" % error)

    def test_add_spatial_view_queries_threads(self):
        diff_nodes = self.input.param("diff-nodes", False)
        query_threads = []
        for i in xrange(len(self.servers)):
            node = (self.master, self.servers[i])[diff_nodes]
            self.query_and_verify_result(self.docs, self.params, node=node)
            q_thread = Thread(target=self.query_and_verify_result,
                                   name="query_thread" + str(i),
                                   args=([self.docs, self.params, node]))
            query_threads.append(q_thread)
            q_thread.start()
        for q_thread in query_threads:
            q_thread.join()
        if self.thread_crashed.is_set():
            self.fail("Error occured during run")

    def test_view_queries_during_rebalance(self):
        start_cluster = self.input.param('start-cluster', 1)
        servers_in = self.input.param('servers_in', 0)
        servers_out = self.input.param('servers_out', 0)
        if start_cluster > 1:
            rebalance = self.cluster.async_rebalance(self.servers[:1],
                                                     self.servers[1:start_cluster], [])
            rebalance.result()
        servs_in = []
        servs_out = []
        if servers_in:
            servs_in = self.servers[start_cluster:servers_in + 1]
        if servers_out:
            if start_cluster > 1:
                servs_out = self.servers[1:start_cluster]
                servs_out = servs_out[-servers_out:]
            else:
                servs_out = self.servers[-servers_out:]
        rebalance = self.cluster.async_rebalance(self.servers, servs_in, servs_out)
        self.query_and_verify_result(self.docs, self.params)
        rebalance.result()

    def test_view_queries_node_pending_state(self):
        operation = self.input.param('operation', 'add_node')
        rest = RestConnection(self.master)
        if operation == 'add_node':
            self.log.info("adding the node %s:%s" % (
                        self.servers[1].ip, self.servers[1].port))
            otpNode = rest.add_node(self.master.rest_username, self.master.rest_password,
                                    self.servers[1].ip, self.servers[1].port)
        elif operation == 'failover':
            nodes = rest.node_statuses()
            nodes = [node for node in nodes
                     if node.ip != self.master.ip or node.port != self.master.port]
            rest.fail_over(nodes[0].id)
        else:
            self.fail("There is no operation %s" % operation)
        self.query_and_verify_result(self.docs, self.params)

    def test_view_queries_failover(self):
        num_nodes = self.input.param('num-nodes', 1)
        self.cluster.failover(self.servers,
#.........这里部分代码省略.........

# 需要导入模块: from membase.helper.spatial_helper import SpatialHelper [as 别名]
# 或者: from membase.helper.spatial_helper.SpatialHelper import insert_docs [as 别名]
class SpatialViewsTests(BaseTestCase):

    def setUp(self):
        super(SpatialViewsTests, self).setUp()
        self.thread_crashed = Event()
        self.thread_stopped = Event()
        self.skip_rebalance = self.input.param("skip_rebalance", False)
        self.use_dev_views = self.input.param("use-dev-views", False)
        self.default_map = "function (doc) {emit(doc.geometry, doc.age);}"
        self.map_updated = "function (doc) {emit(doc.geometry, doc.name);}"
        self.default_ddoc_name = self.input.param("default_ddoc_name", "test-ddoc")
        self.default_view_name = self.input.param("default_view_name", "test-view")
        self.ddoc_op = self.input.param("ddoc-ops", "create") #create\update\delete
        self.bucket_name = "default"
        if self.standard_buckets:
            self.bucket_name = "standard_bucket0"
        if self.sasl_buckets:
            self.bucket_name = "bucket0"
        self.helper = SpatialHelper(self, self.bucket_name)
        if not self.skip_rebalance:
            self.cluster.rebalance(self.servers[:], self.servers[1:], [])
        #load some items to verify
        self.docs = self.helper.insert_docs(self.num_items, 'spatial-doc',
                                            return_docs=True)
        self.num_ddoc = self.input.param('num-ddoc', 1)
        self.views_per_ddoc = self.input.param('views-per-ddoc', 1)
        self.non_spatial_views_per_ddoc = self.input.param('non-spatial-views-per-ddoc', 0)
        if self.ddoc_op == 'update' or self.ddoc_op == 'delete':
            ddocs =  self.make_ddocs(self.num_ddoc, self.views_per_ddoc,
                                     self.non_spatial_views_per_ddoc)
            self.create_ddocs(ddocs)

    def tearDown(self):
        super(SpatialViewsTests, self).tearDown()

    def test_add_spatial_views(self):
        ddocs =  self.make_ddocs(self.num_ddoc, self.views_per_ddoc, self.non_spatial_views_per_ddoc)
        self.perform_ddoc_ops(ddocs)

    def test_add_spatial_views_case_sensative(self):
        ddoc = DesignDocument(self.default_ddoc_name, [], spatial_views=[
                                  View(self.default_view_name, self.default_map,
                                       dev_view=self.use_dev_views, is_spatial=True),
                                  View(self.default_view_name.upper(), self.default_map,
                                       dev_view=self.use_dev_views, is_spatial=True)])
        self.create_ddocs([ddoc])

    def test_add_single_spatial_view(self):
        name_lenght = self.input.param('name_lenght', None)
        view_name = self.input.param('view_name', self.default_view_name)
        if name_lenght:
            view_name = ''.join(random.choice(string.lowercase) for x in xrange(name_lenght))
        not_compilable = self.input.param('not_compilable', False)
        error = self.input.param('error', None)
        map_fn = (self.default_map, 'function (doc) {emit(doc.geometry, doc.age);')[not_compilable]

        ddoc = DesignDocument(self.default_ddoc_name, [], spatial_views=[
                                  View(view_name, map_fn,
                                  dev_view=self.use_dev_views, is_spatial=True)])
        try:
            self.create_ddocs([ddoc])
        except Exception as ex:
            if error and str(ex).find(error) != -1:
                self.log.info("Error caught as expected %s" % error)
                return
            else:
                self.fail("Unexpected error appeared during run %s" % ex)
        if error:
                self.fail("Expected error '%s' didn't appear" % error)

    def test_add_views_to_1_ddoc(self):
        same_names = self.input.param('same-name', False)
        error = self.input.param('error', None)
        num_views_per_ddoc = 10
        create_threads = []
        try:
            for i in xrange(num_views_per_ddoc):
                ddoc = DesignDocument(self.default_ddoc_name, [], spatial_views=[
                                      View(self.default_view_name + (str(i), "")[same_names],
                                           self.default_map,
                                           dev_view=self.use_dev_views, is_spatial=True)])
                create_thread = Thread(target=self.create_ddocs,
                                       name="create_thread" + str(i),
                                       args=([ddoc,],))
                create_threads.append(create_thread)
                create_thread.start()
            for create_thread in create_threads:
                create_thread.join()
        except Exception as ex:
            if error and str(ex).find(error) != -1:
               self.log.info("Error caught as expected %s" % error)
               return
            else:
               self.fail("Unexpected error appeared during run %s" % ex)
        if error:
            self.fail("Expected error '%s' didn't appear" % error)

    def test_add_spatial_views_threads(self):
        same_names = self.input.param('same-name', False)
        num_views_per_ddoc = 10
#.........这里部分代码省略.........

# 需要导入模块: from membase.helper.spatial_helper import SpatialHelper [as 别名]
# 或者: from membase.helper.spatial_helper.SpatialHelper import insert_docs [as 别名]
class SpatialRebalanceTests(unittest.TestCase):
    def setUp(self):
        self.log = logger.Logger.get_logger()
        self.helper = SpatialHelper(self, "default")
        # Setup, but don't rebalance cluster
        self.helper.setup_cluster(False)


    def tearDown(self):
        self.log.info("tear down test")
        self.helper.cleanup_cluster()


    def test_insert_x_delete_y_docs_create_cluster(self):
        num_docs = self.helper.input.param("num-docs", 100000)
        num_deleted_docs = self.helper.input.param("num-deleted-docs", 10000)
        msg = "description : have a single node, insert {0} docs, "\
            "delete {1} docs while creating a cluster and query it"
        self.log.info(msg.format(num_docs, num_deleted_docs))
        design_name = "dev_test_delete_10k_docs_create_cluster"
        prefix = str(uuid.uuid4())[:7]

        # Make sure we are fully de-clustered
        ClusterOperationHelper.cleanup_cluster(self.helper.servers)

        self.helper.create_index_fun(design_name, prefix)
        inserted_keys = self.helper.insert_docs(num_docs, prefix)

        # Start creating the cluster and rebalancing it without waiting until
        # it's finished
        ClusterOperationHelper.add_and_rebalance(self.helper.servers, False)

        deleted_keys = self.helper.delete_docs(num_deleted_docs, prefix)
        self._wait_for_rebalance()

        # Verify that the docs got delete and are no longer part of the
        # spatial view
        results = self.helper.get_results(design_name, num_docs)
        result_keys = self.helper.get_keys(results)
        self.assertEqual(len(result_keys), num_docs - len(deleted_keys))
        self.helper.verify_result(inserted_keys, deleted_keys + result_keys)


    def test_insert_x_delete_y_docs_destroy_cluster(self):
        num_docs = self.helper.input.param("num-docs", 100000)
        num_deleted_docs = self.helper.input.param("num-deleted-docs", 10000)
        msg = "description : have a cluster, insert {0} docs, delete "\
            "{1} docs while destroying the cluster into a single node "\
            "and query it"
        self.log.info(msg.format(num_docs, num_deleted_docs))
        design_name = "dev_test_delete_{0}_docs_destroy_cluster".format(
            num_deleted_docs)
        prefix = str(uuid.uuid4())[:7]

        # Make sure we are fully clustered
        ClusterOperationHelper.add_and_rebalance(self.helper.servers)

        self.helper.create_index_fun(design_name, prefix)
        inserted_keys = self.helper.insert_docs(num_docs, prefix)

        # Start destroying the cluster and rebalancing it without waiting
        # until it's finished
        ClusterOperationHelper.cleanup_cluster(self.helper.servers,
                                                    False)

        deleted_keys = self.helper.delete_docs(num_deleted_docs, prefix)
        self._wait_for_rebalance()

        # Verify that the docs got delete and are no longer part of the
        # spatial view
        results = self.helper.get_results(design_name, num_docs)
        result_keys = self.helper.get_keys(results)
        self.assertEqual(len(result_keys), num_docs - len(deleted_keys))
        self.helper.verify_result(inserted_keys, deleted_keys + result_keys)


    def test_insert_x_docs_during_rebalance(self):
        num_docs = self.helper.input.param("num-docs", 100000)
        msg = "description : have a single node, insert {0} docs, "\
            "query it, add another node, start rebalancing, insert {0} "\
            "docs, finish rebalancing, keep on adding nodes..."
        self.log.info(msg.format(num_docs))
        design_name = "dev_test_insert_{0}_docs_during_rebalance".format(
            num_docs)
        prefix = str(uuid.uuid4())[:7]

        # Make sure we are fully de-clustered
        ClusterOperationHelper.cleanup_cluster(self.helper.servers)

        self.helper.create_index_fun(design_name)
        inserted_keys = self.helper.insert_docs(num_docs, prefix)

        # Add all servers to the master server one by one and start
        # rebalacing
        for server in self.helper.servers[1:]:
            ClusterOperationHelper.add_and_rebalance(
                [self.helper.master, server], False)
            # Docs with the same prefix are overwritten and not newly created
            prefix = str(uuid.uuid4())[:7]
            inserted_keys.extend(self.helper.insert_docs(
#.........这里部分代码省略.........

# 需要导入模块: from membase.helper.spatial_helper import SpatialHelper [as 别名]
# 或者: from membase.helper.spatial_helper.SpatialHelper import insert_docs [as 别名]
class SpatialQueryErrorsTests(BaseTestCase):
    def setUp(self):
        try:
            if 'first_case' not in TestInputSingleton.input.test_params:
                TestInputSingleton.input.test_params['default_bucket'] = False
                TestInputSingleton.input.test_params['skip_cleanup'] = True
                TestInputSingleton.input.test_params['skip_buckets_handle'] = True
            self.default_bucket_name = 'default'
            super(SpatialQueryErrorsTests, self).setUp()
            if 'first_case' in TestInputSingleton.input.test_params:
                self.cluster.rebalance(self.servers[:], self.servers[1:], [])
            # We use only one bucket in this test suite
            self.rest = RestConnection(self.master)
            self.bucket = self.rest.get_bucket(Bucket(name=self.default_bucket_name))
            # num_docs must be a multiple of the number of vbuckets
            self.num_docs = self.input.param("num_docs", 2000)
            # `testname` is used for the design document name as wel as the
            # spatial function name
            self.testname = 'query-errors'
            self.helper = SpatialHelper(self, "default")
            if 'first_case' in TestInputSingleton.input.test_params:
                self.create_ddoc()
                self.helper.insert_docs(self.num_docs, self.testname)
        except Exception as ex:
            self.input.test_params["stop-on-failure"] = True
            self.log.error("SETUP WAS FAILED. ALL TESTS WILL BE SKIPPED")
            self.fail(ex)

    def tearDown(self):
        # clean up will only performed on the last run
        if 'last_case' in TestInputSingleton.input.test_params:
            TestInputSingleton.input.test_params['skip_cleanup'] = False
            TestInputSingleton.input.test_params['skip_buckets_handle'] = False
            super(SpatialQueryErrorsTests, self).tearDown()
        else:
            self.cluster.shutdown(force=True)
            self._log_finish(self)

    def test_query_errors(self):
        all_params = ['skip', 'limit', 'stale', 'bbox', 'start_range',
                      'end_range']
        query_params = {}
        for key in self.input.test_params:
            if key in all_params:
                query_params[key] = str(self.input.test_params[key])

        try:
            self.spatial_query(query_params)
        except QueryViewException as ex:
            self.assertEquals(self.input.test_params['error'],
                              json.loads(ex.reason)['error'])
        else:
            self.fail("Query did not fail, but should have. "
                      "Query parameters were: {0}".format(query_params))


    def create_ddoc(self):
        view_fn = '''function (doc) {
    if (doc.age !== undefined || doc.height !== undefined ||
            doc.bloom !== undefined || doc.shed_leaves !== undefined) {
        emit([doc.age, doc.height, [doc.bloom, doc.shed_leaves]], doc.name);
    }}'''
        self.helper.create_index_fun(self.testname, view_fn)

    def spatial_query(self, params={}, ddoc='test'):
       bucket = self.default_bucket_name
       if not 'stale' in params:
           params['stale'] = 'false'
       return self.rest.query_view(self.testname, self.testname, bucket,
                                   params, type="spatial")

# 需要导入模块: from membase.helper.spatial_helper import SpatialHelper [as 别名]
# 或者: from membase.helper.spatial_helper.SpatialHelper import insert_docs [as 别名]
class SpatialViewTests(unittest.TestCase):
    def setUp(self):
        self.log = logger.Logger.get_logger()
        self.helper = SpatialHelper(self, "default")
        self.helper.setup_cluster()


    def tearDown(self):
        self.helper.cleanup_cluster()


    def test_create_multiple_development_spatial(self):
        self.log.info("description : create multiple spatial views without "
                      "running any spatial view query")
        rest = self.helper.rest
        bucket = self.helper.bucket
        prefix = str(uuid.uuid4())
        name = "dev_test_spatial_multiple"

        design_names = ["{0}-{1}-{2}".format(name, i, prefix) \
                             for i in range(0, 5)]
        for design_name in design_names:
            self.helper.create_index_fun(design_name)
            response = rest.get_spatial(bucket, design_name)
            self.assertTrue(response)
            self.assertEquals(response["_id"],
                              "_design/{0}".format(design_name))
            self.log.info(response)


    def test_insert_x_docs(self):
        num_docs = self.helper.input.param("num-docs", 100)
        self.log.info("description : create a spatial view on {0} documents"\
                          .format(num_docs))
        design_name = "dev_test_insert_{0}_docs".format(num_docs)
        prefix = str(uuid.uuid4())[:7]

        inserted_keys = self._setup_index(design_name, num_docs, prefix)
        self.assertEqual(len(inserted_keys), num_docs)


    # Does verify the full docs and not only the keys
    def test_insert_x_docs_full_verification(self):
        num_docs = self.helper.input.param("num-docs", 100)
        self.log.info("description : create a spatial view with {0} docs"
                      " and verify the full documents".format(num_docs))
        design_name = "dev_test_insert_{0}_docs_full_verification"\
            .format(num_docs)
        prefix = str(uuid.uuid4())[:7]

        self.helper.create_index_fun(design_name)
        inserted_docs = self.helper.insert_docs(num_docs, prefix,
                                                return_docs=True)
        self.helper.query_index_for_verification(design_name, inserted_docs,
                                                 full_docs=True)


    def test_insert_x_delete_y_docs(self):
        num_docs = self.helper.input.param("num-docs", 15000)
        num_deleted_docs = self.helper.input.param("num-deleted-docs", 10000)
        self.log.info("description : create spatial view with {0} docs "
                      " and delete {1} docs".format(num_docs,
                                                    num_deleted_docs))
        design_name = "dev_test_insert_{0}_delete_{1}_docs"\
            .format(num_docs, num_deleted_docs)
        prefix = str(uuid.uuid4())[:7]

        inserted_keys = self._setup_index(design_name, num_docs, prefix)

        # Delete documents and very that the documents got deleted
        deleted_keys = self.helper.delete_docs(num_deleted_docs, prefix)
        results = self.helper.get_results(design_name, 2*num_docs)
        result_keys = self.helper.get_keys(results)
        self.assertEqual(len(result_keys), num_docs-len(deleted_keys))
        self.helper.verify_result(inserted_keys, deleted_keys + result_keys)


    def test_insert_x_update_y_docs(self):
        num_docs = self.helper.input.param("num-docs", 15000)
        num_updated_docs = self.helper.input.param("num-updated-docs", 100)
        self.log.info("description : create spatial view with {0} docs "
                      " and update {1} docs".format(num_docs,
                                                    num_updated_docs))
        design_name = "dev_test_insert_{0}_delete_{1}_docs"\
            .format(num_docs, num_updated_docs)
        prefix = str(uuid.uuid4())[:7]

        self._setup_index(design_name, num_docs, prefix)

        # Update documents and verify that the documents got updated
        updated_keys = self.helper.insert_docs(num_updated_docs, prefix,
                                               dict(updated=True))
        results = self.helper.get_results(design_name, 2*num_docs)
        result_updated_keys = self._get_updated_docs_keys(results)
        self.assertEqual(len(updated_keys), len(result_updated_keys))
        self.helper.verify_result(updated_keys, result_updated_keys)


    def test_get_spatial_during_x_min_load_y_working_set(self):
        num_docs = self.helper.input.param("num-docs", 10000)
#.........这里部分代码省略.........