Python dse_cluster.DseCluster类代码示例

    def load(path, name):
        cluster_path = os.path.join(path, name)
        filename = os.path.join(cluster_path, 'cluster.conf')
        with open(filename, 'r') as f:
            data = yaml.load(f)
        try:
            install_dir = None
            if 'install_dir' in data:
                install_dir = data['install_dir']
                repository.validate(install_dir)
            if install_dir is None and 'cassandra_dir' in data:
                install_dir = data['cassandra_dir']
                repository.validate(install_dir)

            cassandra_version = None
            if 'cassandra_version' in data:
                cassandra_version = LooseVersion(data['cassandra_version'])

            if common.isDse(install_dir):
                cluster = DseCluster(path, data['name'], install_dir=install_dir, create_directory=False, derived_cassandra_version=cassandra_version)
            else:
                cluster = Cluster(path, data['name'], install_dir=install_dir, create_directory=False, derived_cassandra_version=cassandra_version)
            node_list = data['nodes']
            seed_list = data['seeds']
            if 'partitioner' in data:
                cluster.partitioner = data['partitioner']
            if 'config_options' in data:
                cluster._config_options = data['config_options']
            if 'dse_config_options' in data:
                cluster._dse_config_options = data['dse_config_options']
            if 'misc_config_options' in data:
                cluster._misc_config_options = data['misc_config_options']
            if 'log_level' in data:
                cluster.__log_level = data['log_level']
            if 'use_vnodes' in data:
                cluster.use_vnodes = data['use_vnodes']
            if 'datadirs' in data:
                cluster.data_dir_count = int(data['datadirs'])
            extension.load_from_cluster_config(cluster, data)
        except KeyError as k:
            raise common.LoadError("Error Loading " + filename + ", missing property:" + k)

        for node_name in node_list:
            cluster.nodes[node_name] = Node.load(cluster_path, node_name, cluster)
        for seed in seed_list:
            cluster.seeds.append(seed)

        return cluster

    def load(path, name):
        cluster_path = os.path.join(path, name)
        filename = os.path.join(cluster_path, 'cluster.conf')
        with open(filename, 'r') as f:
            data = yaml.load(f)
        try:
            install_dir = None
            if 'install_dir' in data:
                install_dir = data['install_dir']
                repository.validate(install_dir)
            if install_dir is None and 'cassandra_dir' in data:
                install_dir = data['cassandra_dir']
                repository.validate(install_dir)

            if common.isDse(install_dir):
                cluster = DseCluster(path, data['name'], install_dir=install_dir, create_directory=False)
            else:
                cluster = Cluster(path, data['name'], install_dir=install_dir, create_directory=False)
            node_list = data['nodes']
            seed_list = data['seeds']
            if 'partitioner' in data:
                cluster.partitioner = data['partitioner']
            if 'config_options' in data:
                cluster._config_options = data['config_options']
            if 'log_level' in data:
                cluster.__log_level = data['log_level']
            if 'use_vnodes' in data:
                cluster.use_vnodes = data['use_vnodes']
        except KeyError as k:
            raise common.LoadError("Error Loading " + filename + ", missing property:" + k)

        for node_name in node_list:
            cluster.nodes[node_name] = Node.load(cluster_path, node_name, cluster)
        for seed_name in seed_list:
            cluster.seeds.append(cluster.nodes[seed_name])

        return cluster

def use_cluster(cluster_name, nodes, ipformat=None, start=True, workloads=[]):
    global CCM_CLUSTER
    if USE_CASS_EXTERNAL:
        if CCM_CLUSTER:
            log.debug("Using external CCM cluster {0}".format(CCM_CLUSTER.name))
        else:
            log.debug("Using unnamed external cluster")
        return

    if is_current_cluster(cluster_name, nodes):
        log.debug("Using existing cluster, matching topology: {0}".format(cluster_name))
    else:
        if CCM_CLUSTER:
            log.debug("Stopping existing cluster, topology mismatch: {0}".format(CCM_CLUSTER.name))
            CCM_CLUSTER.stop()

        try:
            CCM_CLUSTER = CCMClusterFactory.load(path, cluster_name)
            log.debug("Found existing CCM cluster, {0}; clearing.".format(cluster_name))
            CCM_CLUSTER.clear()
            CCM_CLUSTER.set_install_dir(**CCM_KWARGS)
        except Exception:
            ex_type, ex, tb = sys.exc_info()
            log.warn("{0}: {1} Backtrace: {2}".format(ex_type.__name__, ex, traceback.extract_tb(tb)))
            del tb

            log.debug("Creating new CCM cluster, {0}, with args {1}".format(cluster_name, CCM_KWARGS))
            if DSE_VERSION:
                log.error("creating dse cluster")
                CCM_CLUSTER = DseCluster(path, cluster_name, **CCM_KWARGS)
            else:
                CCM_CLUSTER = CCMCluster(path, cluster_name, **CCM_KWARGS)
            CCM_CLUSTER.set_configuration_options({'start_native_transport': True})
            if CASSANDRA_VERSION >= '2.2':
                CCM_CLUSTER.set_configuration_options({'enable_user_defined_functions': True})
                if CASSANDRA_VERSION >= '3.0':
                    CCM_CLUSTER.set_configuration_options({'enable_scripted_user_defined_functions': True})
            if 'spark' in workloads:
                config_options = {"initial_spark_worker_resources": 0.1}
                CCM_CLUSTER.set_dse_configuration_options(config_options)
            common.switch_cluster(path, cluster_name)
            CCM_CLUSTER.populate(nodes, ipformat=ipformat)
    try:
        jvm_args = []
        # This will enable the Mirroring query handler which will echo our custom payload k,v pairs back
        if PROTOCOL_VERSION >= 4:
            jvm_args = [" -Dcassandra.custom_query_handler_class=org.apache.cassandra.cql3.CustomPayloadMirroringQueryHandler"]
        if(len(workloads) > 0):
            for node in CCM_CLUSTER.nodes.values():
                node.set_workloads(workloads)
        if start:
            log.debug("Starting CCM cluster: {0}".format(cluster_name))
            CCM_CLUSTER.start(wait_for_binary_proto=True, wait_other_notice=True, jvm_args=jvm_args)
            # Added to wait for slow nodes to start up
            for node in CCM_CLUSTER.nodes.values():
                wait_for_node_socket(node, 120)
            setup_keyspace(ipformat=ipformat)
    except Exception:
        log.exception("Failed to start CCM cluster; removing cluster.")

        if os.name == "nt":
            if CCM_CLUSTER:
                for node in CCM_CLUSTER.nodes.itervalues():
                    os.system("taskkill /F /PID " + str(node.pid))
        else:
            call(["pkill", "-9", "-f", ".ccm"])
        remove_cluster()
        raise

    def run(self):
        try:
            if self.options.dse or (not self.options.version and common.isDse(self.options.install_dir)):
                cluster = DseCluster(
                    self.path,
                    self.name,
                    install_dir=self.options.install_dir,
                    version=self.options.version,
                    dse_username=self.options.dse_username,
                    dse_password=self.options.dse_password,
                    opscenter=self.options.opscenter,
                    verbose=True,
                )
            else:
                cluster = Cluster(
                    self.path,
                    self.name,
                    install_dir=self.options.install_dir,
                    version=self.options.version,
                    verbose=True,
                )
        except OSError as e:
            cluster_dir = os.path.join(self.path, self.name)
            import traceback

            print_("Cannot create cluster: %s\n%s" % (str(e), traceback.format_exc()), file=sys.stderr)
            exit(1)

        if self.options.partitioner:
            cluster.set_partitioner(self.options.partitioner)

        if cluster.cassandra_version() >= "1.2.5":
            self.options.binary_protocol = True
        if self.options.binary_protocol:
            cluster.set_configuration_options({"start_native_transport": True})

        if cluster.cassandra_version() >= "1.2" and self.options.vnodes:
            cluster.set_configuration_options({"num_tokens": 256})

        if not self.options.no_switch:
            common.switch_cluster(self.path, self.name)
            print_("Current cluster is now: %s" % self.name)

        if not (self.options.ipprefix or self.options.ipformat):
            self.options.ipformat = "127.0.0.%d"

        if self.options.ssl_path:
            cluster.enable_ssl(self.options.ssl_path, self.options.require_client_auth)

        if self.options.node_ssl_path:
            cluster.enable_internode_ssl(self.options.node_ssl_path)

        if self.nodes is not None:
            try:
                if self.options.debug_log:
                    cluster.set_log_level("DEBUG")
                if self.options.trace_log:
                    cluster.set_log_level("TRACE")
                cluster.populate(
                    self.nodes,
                    self.options.debug,
                    use_vnodes=self.options.vnodes,
                    ipprefix=self.options.ipprefix,
                    ipformat=self.options.ipformat,
                )
                if self.options.start_nodes:
                    profile_options = None
                    if self.options.profile:
                        profile_options = {}
                        if self.options.profile_options:
                            profile_options["options"] = self.options.profile_options
                    if (
                        cluster.start(
                            verbose=self.options.debug_log,
                            wait_for_binary_proto=self.options.binary_protocol,
                            jvm_args=self.options.jvm_args,
                            profile_options=profile_options,
                        )
                        is None
                    ):
                        details = ""
                        if not self.options.debug_log:
                            details = " (you can use --debug-log for more information)"
                        print_("Error starting nodes, see above for details%s" % details, file=sys.stderr)
            except common.ArgumentError as e:
                print_(str(e), file=sys.stderr)
                exit(1)

def use_cluster(cluster_name, nodes, ipformat=None, start=True, workloads=[], set_keyspace=True, ccm_options=None,
                configuration_options={}, dse_cluster=False, dse_options={},
                dse_version=None):
    if (dse_version and not dse_cluster):
        raise ValueError('specified dse_version {} but not dse_cluster'.format(dse_version))
    set_default_cass_ip()

    if ccm_options is None and dse_cluster:
        ccm_options = {"version": dse_version or DSE_VERSION}
    elif ccm_options is None:
        ccm_options = CCM_KWARGS.copy()

    cassandra_version = ccm_options.get('version', CCM_VERSION)
    dse_version = ccm_options.get('version', DSE_VERSION)

    if 'version' in ccm_options:
        ccm_options['version'] = ccm_options['version'].base_version

    global CCM_CLUSTER
    if USE_CASS_EXTERNAL:
        if CCM_CLUSTER:
            log.debug("Using external CCM cluster {0}".format(CCM_CLUSTER.name))
        else:
            log.debug("Using unnamed external cluster")
        if set_keyspace and start:
            setup_keyspace(ipformat=ipformat, wait=False)
        return

    if is_current_cluster(cluster_name, nodes):
        log.debug("Using existing cluster, matching topology: {0}".format(cluster_name))
    else:
        if CCM_CLUSTER:
            log.debug("Stopping existing cluster, topology mismatch: {0}".format(CCM_CLUSTER.name))
            CCM_CLUSTER.stop()

        try:
            CCM_CLUSTER = CCMClusterFactory.load(path, cluster_name)
            log.debug("Found existing CCM cluster, {0}; clearing.".format(cluster_name))
            CCM_CLUSTER.clear()
            CCM_CLUSTER.set_install_dir(**ccm_options)
            CCM_CLUSTER.set_configuration_options(configuration_options)
        except Exception:
            ex_type, ex, tb = sys.exc_info()
            log.warning("{0}: {1} Backtrace: {2}".format(ex_type.__name__, ex, traceback.extract_tb(tb)))
            del tb

            ccm_options.update(cmd_line_args_to_dict('CCM_ARGS'))

            log.debug("Creating new CCM cluster, {0}, with args {1}".format(cluster_name, ccm_options))

            # Make sure we cleanup old cluster dir if it exists
            cluster_path = os.path.join(path, cluster_name)
            if os.path.exists(cluster_path):
                shutil.rmtree(cluster_path)

            if dse_cluster:
                CCM_CLUSTER = DseCluster(path, cluster_name, **ccm_options)
                CCM_CLUSTER.set_configuration_options({'start_native_transport': True})
                CCM_CLUSTER.set_configuration_options({'batch_size_warn_threshold_in_kb': 5})
                if dse_version >= Version('5.0'):
                    CCM_CLUSTER.set_configuration_options({'enable_user_defined_functions': True})
                    CCM_CLUSTER.set_configuration_options({'enable_scripted_user_defined_functions': True})
                if 'spark' in workloads:
                    config_options = {"initial_spark_worker_resources": 0.1}
                    CCM_CLUSTER.set_dse_configuration_options(config_options)
                common.switch_cluster(path, cluster_name)
                CCM_CLUSTER.set_configuration_options(configuration_options)
                CCM_CLUSTER.populate(nodes, ipformat=ipformat)

                CCM_CLUSTER.set_dse_configuration_options(dse_options)
            else:
                CCM_CLUSTER = CCMCluster(path, cluster_name, **ccm_options)
                CCM_CLUSTER.set_configuration_options({'start_native_transport': True})
                if cassandra_version >= Version('2.2'):
                    CCM_CLUSTER.set_configuration_options({'enable_user_defined_functions': True})
                    if cassandra_version >= Version('3.0'):
                        CCM_CLUSTER.set_configuration_options({'enable_scripted_user_defined_functions': True})
                common.switch_cluster(path, cluster_name)
                CCM_CLUSTER.set_configuration_options(configuration_options)
                CCM_CLUSTER.populate(nodes, ipformat=ipformat)

    try:
        jvm_args = []
        # This will enable the Mirroring query handler which will echo our custom payload k,v pairs back

        if 'graph' not in workloads:
            if PROTOCOL_VERSION >= 4:
                jvm_args = [" -Dcassandra.custom_query_handler_class=org.apache.cassandra.cql3.CustomPayloadMirroringQueryHandler"]
        if(len(workloads) > 0):
            for node in CCM_CLUSTER.nodes.values():
                node.set_workloads(workloads)
        if start:
            log.debug("Starting CCM cluster: {0}".format(cluster_name))
            CCM_CLUSTER.start(wait_for_binary_proto=True, wait_other_notice=True, jvm_args=jvm_args)
            # Added to wait for slow nodes to start up
            for node in CCM_CLUSTER.nodes.values():
                wait_for_node_socket(node, 120)
            if set_keyspace:
                setup_keyspace(ipformat=ipformat)
    except Exception:
#.........这里部分代码省略.........