Python cluster.Cluster类代码示例

    def _get_cluster(self, name='test'):
        if self._preserve_cluster and hasattr(self, 'cluster'):
            return self.cluster
        self.test_path = tempfile.mkdtemp(prefix='dtest-')
        # ccm on cygwin needs absolute path to directory - it crosses from cygwin space into
        # regular Windows space on wmic calls which will otherwise break pathing
        if sys.platform == "cygwin":
            self.test_path = subprocess.Popen(["cygpath", "-m", self.test_path], stdout=subprocess.PIPE, stderr=subprocess.STDOUT).communicate()[0].rstrip()
        debug("cluster ccm directory: " + self.test_path)
        version = os.environ.get('CASSANDRA_VERSION')
        cdir = CASSANDRA_DIR

        if version:
            cluster = Cluster(self.test_path, name, cassandra_version=version)
        else:
            cluster = Cluster(self.test_path, name, cassandra_dir=cdir)

        if DISABLE_VNODES:
            cluster.set_configuration_options(values={'num_tokens': None})
        else:
            cluster.set_configuration_options(values={'initial_token': None, 'num_tokens': NUM_TOKENS})

        if OFFHEAP_MEMTABLES:
            cluster.set_configuration_options(values={'memtable_allocation_type': 'offheap_objects'})

        cluster.set_datadir_count(DATADIR_COUNT)

        return cluster

def setup_module(cls):
    validate_ccm_viable()
    validate_host_viable()
    cls.ccm_cluster = object()
    teardown_package()
    try:
        try:
            ccm_cluster = CCMCluster.load(path, IPV6_CLUSTER_NAME)
            log.debug("Found existing ccm test ipv6 cluster, clearing")
            ccm_cluster.clear()
        except Exception:
            log.debug("Creating new ccm test ipv6 cluster")
            if CASSANDRA_DIR:
                ccm_cluster = CCMCluster(path, IPV6_CLUSTER_NAME, cassandra_dir=CASSANDRA_DIR)
            else:
                ccm_cluster = CCMCluster(path, IPV6_CLUSTER_NAME, cassandra_version=CASSANDRA_VERSION)
            ccm_cluster.set_configuration_options({'start_native_transport': True})
            common.switch_cluster(path, IPV6_CLUSTER_NAME)
            ccm_cluster.populate(1, ipformat='::%d')

        log.debug("Starting ccm test cluster")
        ccm_cluster.start(wait_for_binary_proto=True)
    except Exception:
        log.exception("Failed to start ccm cluster:")
        raise

    log.debug("Switched to ipv6 cluster")
    cls.ccm_cluster = ccm_cluster

def setup_package():
    print_('Using Cassandra version: %s' % CASSANDRA_VERSION)
    try:
        try:
            cluster = CCMCluster.load(path, CLUSTER_NAME)
            log.debug("Found existing ccm test cluster, clearing")
            cluster.clear()
            if CASSANDRA_DIR:
                cluster.set_cassandra_dir(cassandra_dir=CASSANDRA_DIR)
            else:
                cluster.set_cassandra_dir(cassandra_version=CASSANDRA_VERSION)
        except Exception:
            if CASSANDRA_DIR:
                log.debug("Creating new ccm test cluster with cassandra dir %s", CASSANDRA_DIR)
                cluster = CCMCluster(path, CLUSTER_NAME, cassandra_dir=CASSANDRA_DIR)
            else:
                log.debug("Creating new ccm test cluster with version %s", CASSANDRA_VERSION)
                cluster = CCMCluster(path, CLUSTER_NAME, cassandra_version=CASSANDRA_VERSION)
            cluster.set_configuration_options({'start_native_transport': True})
            common.switch_cluster(path, CLUSTER_NAME)
            cluster.populate(3)

        log.debug("Starting ccm test cluster")
        cluster.start(wait_for_binary_proto=True, wait_other_notice=True)
    except Exception:
        log.exception("Failed to start ccm cluster:")
        raise

    global CCM_CLUSTER
    CCM_CLUSTER = cluster
    setup_test_keyspace()

def clear_and_use_multidc(dc_list):
    teardown_package()
    try:
        try:
            cluster = CCMCluster.load(path, MULTIDC_CLUSTER_NAME)
            log.debug("Found existing ccm test multi-dc cluster, clearing")
            cluster.clear()
        except Exception:
            log.debug("Creating new ccm test multi-dc cluster")
            if CASSANDRA_DIR:
                cluster = CCMCluster(path, MULTIDC_CLUSTER_NAME, cassandra_dir=CASSANDRA_DIR)
            else:
                cluster = CCMCluster(path, MULTIDC_CLUSTER_NAME, cassandra_version=CASSANDRA_VERSION)
            cluster.set_configuration_options({'start_native_transport': True})
            common.switch_cluster(path, MULTIDC_CLUSTER_NAME)
            cluster.populate(dc_list)

        log.debug("Starting ccm test cluster")
        cluster.start(wait_for_binary_proto=True, wait_other_notice=True)
    except Exception:
        log.exception("Failed to start ccm cluster:")
        raise

    global CCM_CLUSTER
    CCM_CLUSTER = cluster
    setup_test_keyspace()
    log.debug("Switched to multidc cluster")

 def __get_cluster(self, name='test'):
     self.test_path = tempfile.mkdtemp(prefix='dtest-')
     try:
         version = os.environ['CASSANDRA_VERSION']
         cluster = Cluster(self.test_path, name, cassandra_version=version)
     except KeyError:
         try:
             cdir = os.environ['CASSANDRA_DIR']
         except KeyError:
             cdir = DEFAULT_DIR
         cluster = Cluster(self.test_path, name, cassandra_dir=cdir)
     if ENABLE_VNODES:
         cluster.set_configuration_options(values={'initial_token': None, 'num_tokens': 256})
     return cluster

def use_cluster(cluster_name, nodes, ipformat=None, start=True):
    global CCM_CLUSTER
    if USE_CASS_EXTERNAL:
        if CCM_CLUSTER:
            log.debug("Using external CCM cluster {0}".format(CCM_CLUSTER.name))
        else:
            log.debug("Using unnamed external cluster")
        return

    if is_current_cluster(cluster_name, nodes):
        log.debug("Using existing cluster, matching topology: {0}".format(cluster_name))
    else:
        if CCM_CLUSTER:
            log.debug("Stopping existing cluster, topology mismatch: {0}".format(CCM_CLUSTER.name))
            CCM_CLUSTER.stop()

        try:
            CCM_CLUSTER = CCMClusterFactory.load(path, cluster_name)
            log.debug("Found existing CCM cluster, {0}; clearing.".format(cluster_name))
            CCM_CLUSTER.clear()
            CCM_CLUSTER.set_install_dir(**CCM_KWARGS)
        except Exception:
            ex_type, ex, tb = sys.exc_info()
            log.warn("{0}: {1} Backtrace: {2}".format(ex_type.__name__, ex, traceback.extract_tb(tb)))
            del tb

            log.debug("Creating new CCM cluster, {0}, with args {1}".format(cluster_name, CCM_KWARGS))
            CCM_CLUSTER = CCMCluster(path, cluster_name, **CCM_KWARGS)
            CCM_CLUSTER.set_configuration_options({'start_native_transport': True})
            if CASSANDRA_VERSION >= '2.2':
                CCM_CLUSTER.set_configuration_options({'enable_user_defined_functions': True})
            common.switch_cluster(path, cluster_name)
            CCM_CLUSTER.populate(nodes, ipformat=ipformat)
    try:
        jvm_args = []
        # This will enable the Mirroring query handler which will echo our custom payload k,v pairs back
        if PROTOCOL_VERSION >= 4:
            jvm_args = [" -Dcassandra.custom_query_handler_class=org.apache.cassandra.cql3.CustomPayloadMirroringQueryHandler"]

        if start:
            log.debug("Starting CCM cluster: {0}".format(cluster_name))
            CCM_CLUSTER.start(wait_for_binary_proto=True, wait_other_notice=True, jvm_args=jvm_args)
            setup_keyspace(ipformat=ipformat)
    except Exception:
        log.exception("Failed to start CCM cluster; removing cluster.")
        
        if os.name == "nt":
            if CCM_CLUSTER:
                for node in CCM_CLUSTER.nodes.itervalues():
                    os.system("taskkill /F /PID " + str(node.pid))
        else:
            call(["pkill", "-9", "-f", ".ccm"])
        remove_cluster()
        raise

    def setUp(self):
        # cleaning up if a previous execution didn't trigger tearDown (which
        # can happen if it is interrupted by KeyboardInterrupt)
        # TODO: move that part to a generic fixture
        if os.path.exists(LAST_TEST_DIR):
            with open(LAST_TEST_DIR) as f:
                self.test_path = f.readline().strip("\n")
                name = f.readline()
                try:
                    self.cluster = Cluster.load(self.test_path, name)
                    # Avoid waiting too long for node to be marked down
                    self.__cleanup_cluster()
                except IOError:
                    # after a restart, /tmp will be emptied so we'll get an IOError when loading the old cluster here
                    pass

        self.cluster = self.__get_cluster()
        # the failure detector can be quite slow in such tests with quick start/stop
        self.cluster.set_configuration_options(values={"phi_convict_threshold": 5})
        self.cluster.set_configuration_options(values={"rpc_timeout_in_ms": 15000})
        with open(LAST_TEST_DIR, "w") as f:
            f.write(self.test_path + "\n")
            f.write(self.cluster.name)
        if DEBUG:
            self.cluster.set_log_level("DEBUG")
        self.connections = []
        self.runners = []

    def replace_test(self):
        cluster = self.cluster

        tokens = Cluster.balanced_tokens(3)
        cluster.populate(3, tokens=tokens).start()
        [node1, node2, node3] = cluster.nodelist()

        cursor = self.cql_connection(node1).cursor()
        self.create_ks(cursor, 'ks', 3)
        self.create_cf(cursor, 'cf')

        for n in xrange(0, 10000):
            insert_c1c2(cursor, n, "QUORUM")

        cluster.flush()

        node3.stop(wait_other_notice=True)
        time.sleep(.5)

        node4 = new_node(cluster, token=tokens[2])
        node4.start(replace_token=tokens[2])
        time.sleep(.5)
        cluster.cleanup()
        time.sleep(.5)

        for n in xrange(0, 10000):
            query_c1c2(cursor, n, "QUORUM")

        sizes = [ node.data_size() for node in cluster.nodelist() if node.is_running()]
        assert_almost_equal(*sizes)

 def setUp(self):
     '''Create a cluster for cqlsh tests. Assumes that ccmtest.Tester's
     teardown() method will safely stop and remove self.cluster.'''
     self.cluster = Cluster(CLUSTER_PATH, "run_cqlsh",
                            cassandra_version='git:trunk')
     self.cluster.populate(1).start(wait_for_binary_proto=True)
     [self.node] = self.cluster.nodelist()

    def test3(self):
        self.cluster = Cluster(CLUSTER_PATH, "test3", cassandra_version='2.0.3')
        self.cluster.populate(2)
        self.cluster.start()
        self.cluster.cleanup()

        self.cluster.clear()
        self.cluster.stop()

def use_cluster(cluster_name, nodes, ipformat=None, start=True):
    if is_current_cluster(cluster_name, nodes):
        log.debug("Using existing cluster %s", cluster_name)
        return

    global CCM_CLUSTER
    if CCM_CLUSTER:
        log.debug("Stopping cluster %s", CCM_CLUSTER.name)
        CCM_CLUSTER.stop()

    try:
        try:
            cluster = CCMClusterFactory.load(path, cluster_name)
            log.debug("Found existing ccm %s cluster; clearing", cluster_name)
            cluster.clear()
            cluster.set_install_dir(**CCM_KWARGS)
        except Exception:
            log.debug("Creating new ccm %s cluster with %s", cluster_name, CCM_KWARGS)
            cluster = CCMCluster(path, cluster_name, **CCM_KWARGS)
            cluster.set_configuration_options({'start_native_transport': True})
            common.switch_cluster(path, cluster_name)
            cluster.populate(nodes, ipformat=ipformat)

        if start:
            log.debug("Starting ccm %s cluster", cluster_name)
            cluster.start(wait_for_binary_proto=True, wait_other_notice=True)
            setup_test_keyspace()

        CCM_CLUSTER = cluster
    except Exception:
        log.exception("Failed to start ccm cluster:")
        raise

    def restart_test(self):
        self.cluster = Cluster(CLUSTER_PATH, "restart", cassandra_version='2.0.9')
        self.cluster.populate(3)
        self.cluster.start()

        self.cluster.stop()
        self.cluster.start()

        self.cluster.show(True)

def use_cluster(cluster_name, nodes, ipformat=None, start=True):
    global CCM_CLUSTER
    if USE_CASS_EXTERNAL:
        if CCM_CLUSTER:
            log.debug("Using external ccm cluster %s", CCM_CLUSTER.name)
        else:
            log.debug("Using unnamed external cluster")
        return

    if is_current_cluster(cluster_name, nodes):
        log.debug("Using existing cluster %s", cluster_name)
        return

    if CCM_CLUSTER:
        log.debug("Stopping cluster %s", CCM_CLUSTER.name)
        CCM_CLUSTER.stop()

    try:
        try:
            cluster = CCMClusterFactory.load(path, cluster_name)
            log.debug("Found existing ccm %s cluster; clearing", cluster_name)
            cluster.clear()
            cluster.set_install_dir(**CCM_KWARGS)
        except Exception:
            log.debug("Creating new ccm %s cluster with %s", cluster_name, CCM_KWARGS)
            cluster = CCMCluster(path, cluster_name, **CCM_KWARGS)
            cluster.set_configuration_options({'start_native_transport': True})
            if CASSANDRA_VERSION >= '2.2':
                cluster.set_configuration_options({'enable_user_defined_functions': True})
            common.switch_cluster(path, cluster_name)
            cluster.populate(nodes, ipformat=ipformat)

        jvm_args = []
        # This will enable the Mirroring query handler which will echo our custom payload k,v pairs back
        if PROTOCOL_VERSION >= 4:
            jvm_args = [" -Dcassandra.custom_query_handler_class=org.apache.cassandra.cql3.CustomPayloadMirroringQueryHandler"]

        if start:
            log.debug("Starting ccm %s cluster", cluster_name)
            cluster.start(wait_for_binary_proto=True, wait_other_notice=True, jvm_args=jvm_args)
            setup_keyspace(ipformat=ipformat)

        CCM_CLUSTER = cluster
    except Exception:
        log.exception("Failed to start ccm cluster. Removing cluster.")
        remove_cluster()
        call(["pkill", "-9", "-f", ".ccm"])
        raise

 def _load_current_cluster(self):
     name = common.current_cluster_name(self.path)
     if name is None:
         print_('No currently active cluster (use ccm cluster switch)')
         exit(1)
     try:
         return Cluster.load(self.path, name)
     except common.LoadError as e:
         print_(str(e))
         exit(1)

class IntegrationTest(TestCase):

    def setUp(self):
        self.test_path = tempfile.mkdtemp(prefix=self.name)
        print
        print "CCM using %s" % self.test_path

        # ccm setup
        self.session = None
        shutil.rmtree(self.test_path)
        os.makedirs(self.test_path)
        self.cluster = Cluster(self.test_path, "test", cassandra_version="3.0.0")

        # sstable tools setup
        self.uberjar_location = glob.glob("%s/../../../target/sstable-*.jar" % os.path.dirname(os.path.realpath(__file__)))[0]
        print "Using sstable build located at: %s" % self.uberjar_location


    def cql_connection(self, node, keyspace=None, timeout=60):
        if self.session:
            return self.session

        deadline = time.time() + timeout
        while True:
            try:
                self.connection_cluster = PyCluster([node.network_interfaces['binary'][0]],
                                                    port=node.network_interfaces['binary'][1])
                self.session = self.connection_cluster.connect()
                if keyspace:
                    self.session.execute(("CREATE KEYSPACE IF NOT EXISTS %s WITH REPLICATION = " +
                                         "{ 'class' : 'SimpleStrategy', 'replication_factor' : 1 };") % keyspace)
                    self.session.execute("USE %s" % keyspace)

                return self.session
            except NoHostAvailable:
                if time.time() > deadline:
                    raise
                else:
                    time.sleep(0.25)

    def tearDown(self):
        self.cluster.stop()