Python ApiResource.get_all_clusters方法代码示例

# 需要导入模块: from cm_api.api_client import ApiResource [as 别名]
# 或者: from cm_api.api_client.ApiResource import get_all_clusters [as 别名]
def adjust_yarn_memory_limits(region, stack_name, restart=True):
    ec2_conn = create_ec2_connection(region)
    manager_instance = get_manager_instance(ec2_conn, stack_name)
    with cm_tunnel_ctx(manager_instance) as local_port:
        cm_api = ApiResource('localhost', username='admin', password='admin',
                             server_port=local_port, version=9)
        cluster = list(cm_api.get_all_clusters())[0]
        host = list(cm_api.get_all_hosts())[0]  # all hosts same instance type
        yarn = filter(lambda x: x.type == 'YARN',
                      list(cluster.get_all_services()))[0]
        rm_cg = filter(lambda x: x.roleType == 'RESOURCEMANAGER',
                       list(yarn.get_all_role_config_groups()))[0]
        nm_cg = filter(lambda x: x.roleType == 'NODEMANAGER',
                       list(yarn.get_all_role_config_groups()))[0]
        rm_cg.update_config({
            'yarn_scheduler_maximum_allocation_mb': (
                int(host.totalPhysMemBytes / 1024. / 1024.)),
            'yarn_scheduler_maximum_allocation_vcores': host.numCores})
        nm_cg.update_config({
            'yarn_nodemanager_resource_memory_mb': (
                int(host.totalPhysMemBytes / 1024. / 1024.)),
            'yarn_nodemanager_resource_cpu_vcores': host.numCores})
        cluster.deploy_client_config().wait()
        if restart:
            cluster.restart().wait()

# 需要导入模块: from cm_api.api_client import ApiResource [as 别名]
# 或者: from cm_api.api_client.ApiResource import get_all_clusters [as 别名]
def get_cluster_specs():
    cm_api = ApiResource(os.environ['MANAGER_HOST'], username='admin',
                         password='admin', server_port=7180, version=9)
    host = list(cm_api.get_all_hosts())[0]  # all hosts same instance type
    cluster = list(cm_api.get_all_clusters())[0]
    yarn = filter(lambda x: x.type == 'YARN',
                  list(cluster.get_all_services()))[0]
    return {'num_worker_nodes': len(yarn.get_roles_by_type('NODEMANAGER')),
            'num_cores': host.numCores, 'node_memory': host.totalPhysMemBytes}

# 需要导入模块: from cm_api.api_client import ApiResource [as 别名]
# 或者: from cm_api.api_client.ApiResource import get_all_clusters [as 别名]
def do_call(host, port, version, user, password, cluster_name, parcel_name, parcel_version, parcel_repo, init_pre_dir, init_post_dir):
    api = ApiResource(host, port, user, password, False, version)
    if not parcel_repo.endswith('/'):
        parcel_repo += '/'
    if re.match(REGEX_VERSION, parcel_version) is None or re.match(REGEX_VERSION, parcel_version).group() != parcel_version:
        raise Exception('Parcel [' + parcel_name + '] is qualified by invalid version [' + parcel_version + '] expected to match regular expression [' + REGEX_VERSION + ']')
    if not parcel_repo.endswith(parcel_version + '/'):
        raise Exception('Parcel [' + parcel_name + '] is qualified by invalid version [' + parcel_version + '] when compared with repository [' + parcel_repo + ']')    
    cm_config = api.get_cloudera_manager().get_config(view='full')
    repo_config = cm_config['REMOTE_PARCEL_REPO_URLS']
    repo_list = repo_config.value or repo_config.default
    if parcel_repo not in repo_list:     
        repo_list += ',' + parcel_repo
        api.get_cloudera_manager().update_config({'REMOTE_PARCEL_REPO_URLS': repo_list})
        time.sleep(POLL_SEC)  # The parcel synchronize end-point is not exposed via the API, so sleep instead
    cluster_names = []
    if cluster_name is None:
        for cluster in api.get_all_clusters():
            cluster_names.append(cluster.name)
    else:
        cluster_names.append(cluster_name)
    for cluster_name_itr in cluster_names:
        print 'Cluster [DEPLOYMENT] starting ... '
        cluster = api.get_cluster(cluster_name_itr)
        parcel = cluster.get_parcel(parcel_name, parcel_version)
        print 'Parcel [DEPLOYMENT] starting ... '
        do_parcel_op(cluster, parcel_name, parcel_version, 'DOWNLOAD', 'AVAILABLE_REMOTELY', 'DOWNLOADED', 'start_download')
        do_parcel_op(cluster, parcel_name, parcel_version, 'DISTRIBUTE', 'DOWNLOADED', 'DISTRIBUTED', 'start_distribution')
        do_parcel_op(cluster, parcel_name, parcel_version, 'ACTIVATE', 'DISTRIBUTED', 'ACTIVATED', 'activate')
        parcel = cluster.get_parcel(parcel_name, parcel_version)
        if parcel.stage != 'ACTIVATED':
            raise Exception('Parcel is currently mid-stage [' + parcel.stage + '], please wait for this to complete')
        print 'Parcel [DEPLOYMENT] finished'
        if init_pre_dir is not None and os.path.isdir(init_pre_dir):
            print 'Cluster [PRE_INIT] starting ... '
            for script in glob.glob(init_pre_dir + '/*.sh'):
                subprocess.call([script])
            print 'Cluster [PRE_INIT] finihsed'            
        print 'Cluster [CONFIG_DEPLOYMENT] starting ... '
        cluster.deploy_client_config()
        cmd = cluster.deploy_client_config()
        if not cmd.wait(TIMEOUT_SEC).success:
            raise Exception('Failed to deploy client configs')
        print 'Cluster [CONFIG_DEPLOYMENT] finihsed'
        print 'Cluster [STOP] starting ... '
        cluster.stop().wait()
        print 'Cluster [STOP] finihsed'
        print 'Cluster [START] starting ... '
        cluster.start().wait()
        print 'Cluster [START] finihsed'
        if init_post_dir is not None and os.path.isdir(init_post_dir):
            print 'Cluster [POST_INIT] starting ... '
            for script in glob.glob(init_post_dir + '/*.sh'):
                subprocess.call([script])
            print 'Cluster [POST_INIT] finihsed'            
        print 'Cluster [DEPLOYMENT] finished'

# 需要导入模块: from cm_api.api_client import ApiResource [as 别名]
# 或者: from cm_api.api_client.ApiResource import get_all_clusters [as 别名]
 def getClusterInformation(self):
     api = ApiResource(self.cm_host, username=self.user, password=self.passwd)
     logger.info('Received; user -> %s, password -> %s, host -> %s', self.user, self.passwd, self.cm_host)
     for c in api.get_all_clusters():
         clusterInf = "Cluster name %s and version %s" %(c.name, c.version)
         #print "Cluster name %s and version %s" %(c.name, c.version)
         logger.info("Cluster name %s and version %s", c.name, c.version)
         if c.version == "CDH5":
             cdh5 = c
     return cdh5, clusterInf

# 需要导入模块: from cm_api.api_client import ApiResource [as 别名]
# 或者: from cm_api.api_client.ApiResource import get_all_clusters [as 别名]
def find_impala_in_cm(cm_host, cm_user, cm_password, cm_cluster_name):
  """Finds the Impala service in CM and returns an Impala instance."""
  cm = ApiResource(cm_host, username=cm_user, password=cm_password)
  cm_impalas = [service for cluster in cm.get_all_clusters()
                if cm_cluster_name is None or cm_cluster_name == cluster.name
                for service in cluster.get_all_services() if service.type == "IMPALA"]
  if len(cm_impalas) > 1:
    raise Exception("Found %s Impala services in CM;" % len(cm_impalas) +
        " use --cm-cluster-name option to specify which one to use.")
  if len(cm_impalas) == 0:
    raise Exception("No Impala services found in CM")
  return Impala(cm_impalas[0])

# 需要导入模块: from cm_api.api_client import ApiResource [as 别名]
# 或者: from cm_api.api_client.ApiResource import get_all_clusters [as 别名]
def main():
    s,a = arg_handle()
    for i in range(0,15):
        while True:
            try:
                cm_host = "127.0.0.1"
                api = ApiResource(cm_host, username="admin", password="admin")
                cdh=api.get_all_clusters()[0]
            except:
                print "Failed to connect to Cloudera Manager."
                print "Attempting to connect to Cloudera Manager..."
                time.sleep(15)
                continue
            break
    srv=cdh.get_service(s)
    actions[a](srv,s)

# 需要导入模块: from cm_api.api_client import ApiResource [as 别名]
# 或者: from cm_api.api_client.ApiResource import get_all_clusters [as 别名]
def update_cm(cm_host, cm_port, username, password):
    """Update config using the CM API (note: will restart service)"""
    elts = generate_xml_elements()
    cm_api = ApiResource(cm_host, username=username, password=password,
                         server_port=cm_port, version=9)
    cluster = list(cm_api.get_all_clusters())[0]
    hdfs = filter(lambda x: x.type == 'HDFS',
                  list(cluster.get_all_services()))[0]
    print("Updating HFDS core-site.xml safety valve...")
    _ = hdfs.update_config({
        'core_site_safety_valve': '\n'.join(tostring(e) for e in elts)})
    print("Deploying client config across the cluster...")
    cluster.deploy_client_config().wait()
    print("Restarting necessary services...")
    cluster.restart().wait()
    print("Done!")

# 需要导入模块: from cm_api.api_client import ApiResource [as 别名]
# 或者: from cm_api.api_client.ApiResource import get_all_clusters [as 别名]
def main():
    """
    This is a script to export a current Cloudera Manager cluster configuration into an Hadrian supported format.
    You can then use these configuration files as the basis for your new cluster configs.
    """

    parser = argparse.ArgumentParser(description='Export Cloudera Manager configs in an Hadrian friendly format.')
    parser.add_argument('-H', '--host', '--hostname', action='store', dest='hostname', required=True, help='CM Server Name')
    parser.add_argument('-p', '--port', action='store', dest='port', type=int, default=7180, help='CM Port')
    parser.add_argument('-u', '--user', '--username', action='store', dest='username', required=True, help='CM username')
    args = parser.parse_args()

    password = getpass.getpass('Please enter your Cloudera Manager passsword: ')
    api = ApiResource(args.hostname, args.port, args.username, password, version=4)

    for cluster in api.get_all_clusters():
        conf_dir = './confs/' + cluster.name
        if not os.path.exists(conf_dir):
            os.makedirs(conf_dir)

            for service in cluster.get_all_services():
                with open(conf_dir + '/' + service.name + '.ini', 'w') as f:
                    print 'Dumping Service config for ' + service.name
                    rcg = list()

                    for i in service.get_all_role_config_groups():
                        rcg.append(i.name)
                        f.write('[' + service.type + ']\n')
                        f.write('config_groups=' + ','.join(rcg))
                        f.write('\n\n')
                        f.write('[' + service.name + '-svc-config]\n')

                    for item in service.get_config():
                        for k,v in item.iteritems():
                            f.write(k + '=' + str(v) + '\n')

                    for i in service.get_all_role_config_groups():
                        f.write('\n')
                        f.write('[' + i.name + ']\n')
                        for k,v in i.get_config('full').iteritems():
                            if v.value is not None:
                                f.write(k + '=' + str(v.value) + '\n')
                    f.close()
        else:
            print 'Cluster config dir already exists.  Please rename or remove existing config dir: ' + conf_dir

# 需要导入模块: from cm_api.api_client import ApiResource [as 别名]
# 或者: from cm_api.api_client.ApiResource import get_all_clusters [as 别名]
def reset_cm(cm_host, cm_port, username, password):
    """Elim S3 config from CM API safety valve (service restart necessary)"""
    s3_props = set(get_s3_properties())
    cm_api = ApiResource(cm_host, username=username, password=password,
                         server_port=cm_port, version=9)
    cluster = list(cm_api.get_all_clusters())[0]
    hdfs = filter(lambda x: x.type == 'HDFS',
                  list(cluster.get_all_services()))[0]
    print("Getting current safety valve config")
    current_config = hdfs.get_config('full')[0]['core_site_safety_valve'].value
    # need the "<foo>...</foo>" to make it valid XML (bc it requires root elt)
    elts = list(fromstring('<foo>' + current_config + '</foo>'))
    new_elts = filter(lambda x: x.find('name').text not in s3_props, elts)
    print("Updating safety valve and deleting S3 config")
    _ = hdfs.update_config({
        'core_site_safety_valve': '\n'.join(tostring(e) for e in new_elts)})
    print("Deploying client config across the cluster...")
    cluster.deploy_client_config().wait()
    print("Restarting necessary services...")
    cluster.restart().wait()
    print("Done!")

# 需要导入模块: from cm_api.api_client import ApiResource [as 别名]
# 或者: from cm_api.api_client.ApiResource import get_all_clusters [as 别名]
def do_call(host, port, user, password, cluster_name, service_role_name, random_index):
    api = ApiResource(host, port, user, password, False, MAN_API_VERSION);
    for cluster in api.get_all_clusters():
        if cluster_name is None:
            break
        elif cluster_name == cluster.name:
            break
    if cluster_name is not None and cluster_name != cluster.name:
        print >> sys.stderr, "Cloud not find cluster: " + cluster_name
        return -2;
    do_print_header()
    for service in cluster.get_all_services():
        do_print_line_item(api, service, service_role_name, random_index, 'HDFS', 'NAMENODE', 'namenode_port', [], [])
        do_print_line_item(api, service, service_role_name, random_index, 'KUDU', 'KUDU_MASTER', 'webserver_port', [], [])
        do_print_line_item(api, service, service_role_name, random_index, 'HUE', 'HUE_SERVER', 'hue_http_port', [], [])
        do_print_line_item(api, service, service_role_name, random_index, 'HIVE', 'HIVESERVER2', 'hs2_thrift_address_port', [], [])
        do_print_line_item(api, service, service_role_name, random_index, 'IMPALA', 'IMPALAD', 'beeswax_port', [], [])
        do_print_line_item(api, service, service_role_name, random_index, 'FLUME', 'AGENT', 'agent_http_port', [], [])
        do_print_line_item(api, service, service_role_name, random_index, 'KAFKA', 'KAFKA_BROKER', 'port', [], [])
        do_print_line_item(api, service, service_role_name, random_index, 'ZOOKEEPER', 'SERVER', 'clientPort', [], [])
    do_print_footer()

# 需要导入模块: from cm_api.api_client import ApiResource [as 别名]
# 或者: from cm_api.api_client.ApiResource import get_all_clusters [as 别名]
def get_cluster_info(manager_host, server_port=7180, username='admin',
                     password='admin'):
    cm_api = ApiResource(manager_host, username=username, password=password,
                         server_port=server_port, version=9)
    host = list(cm_api.get_all_hosts())[0]  # all hosts same instance type
    cluster = list(cm_api.get_all_clusters())[0]
    yarn = filter(lambda x: x.type == 'YARN',
                  list(cluster.get_all_services()))[0]
    hive = filter(lambda x: x.type == 'HIVE',
                  list(cluster.get_all_services()))[0]
    impala = filter(lambda x: x.type == 'IMPALA',
                    list(cluster.get_all_services()))[0]
    hive_hs2 = hive.get_roles_by_type('HIVESERVER2')[0]
    hive_host = cm_api.get_host(hive_hs2.hostRef.hostId).hostname
    hive_port = int(
        hive_hs2.get_config('full')['hs2_thrift_address_port'].default)
    impala_hs2 = impala.get_roles_by_type('IMPALAD')[0]
    impala_host = cm_api.get_host(impala_hs2.hostRef.hostId).hostname
    impala_port = int(impala_hs2.get_config('full')['hs2_port'].default)
    return {'num_worker_nodes': len(yarn.get_roles_by_type('NODEMANAGER')),
            'node_cores': host.numCores, 'node_memory': host.totalPhysMemBytes,
            'hive_host': hive_host, 'hive_port': hive_port,
            'impala_host': impala_host, 'impala_port': impala_port}

# 需要导入模块: from cm_api.api_client import ApiResource [as 别名]
# 或者: from cm_api.api_client.ApiResource import get_all_clusters [as 别名]
def adjust_yarn_memory_limits(region, stack_name):
    ec2_conn = create_ec2_connection(region)
    manager_instance = get_manager_instance(ec2_conn, stack_name)
    cm_api = ApiResource("localhost", username="admin", password="admin", server_port=64999, version=9)
    with http_tunnel_ctx(manager_instance, 7180, 64999):
        cluster = list(cm_api.get_all_clusters())[0]
        host = list(cm_api.get_all_hosts())[0]  # all hosts same instance type
        yarn = filter(lambda x: x.type == "YARN", list(cluster.get_all_services()))[0]
        rm_cg = filter(lambda x: x.roleType == "RESOURCEMANAGER", list(yarn.get_all_role_config_groups()))[0]
        nm_cg = filter(lambda x: x.roleType == "NODEMANAGER", list(yarn.get_all_role_config_groups()))[0]
        rm_cg.update_config(
            {
                "yarn_scheduler_maximum_allocation_mb": (int(host.totalPhysMemBytes / 1024.0 / 1024.0)),
                "yarn_scheduler_maximum_allocation_vcores": host.numCores,
            }
        )
        nm_cg.update_config(
            {
                "yarn_nodemanager_resource_memory_mb": (int(host.totalPhysMemBytes / 1024.0 / 1024.0)),
                "yarn_nodemanager_resource_cpu_vcores": host.numCores,
            }
        )
        cluster.deploy_client_config().wait()
        cluster.restart().wait()

# 需要导入模块: from cm_api.api_client import ApiResource [as 别名]
# 或者: from cm_api.api_client.ApiResource import get_all_clusters [as 别名]
def main():
  api = ApiResource(host='r2341-d5-us01',user='admin',password='admin')
  #is get cluster None? 
  what = api.get_all_clusters()
  print "what:", what

# 需要导入模块: from cm_api.api_client import ApiResource [as 别名]
# 或者: from cm_api.api_client.ApiResource import get_all_clusters [as 别名]
def main():
  api = ApiResource('r2341-d5-us01',username='admin',password='admin')
  clusters = api.get_all_clusters()
  print "clusters:", clusters
  if len(clusters) == 0: 
    print "none"

# 需要导入模块: from cm_api.api_client import ApiResource [as 别名]
# 或者: from cm_api.api_client.ApiResource import get_all_clusters [as 别名]
def install_java_8(region, stack_name):
    # following general protocol for upgrading to JDK 1.8 here:
    # http://www.cloudera.com/content/cloudera/en/documentation/core/v5-3-x/topics/cdh_cm_upgrading_to_jdk8.html
    ec2_conn = create_ec2_connection(region)
    manager_instance = get_manager_instance(ec2_conn, stack_name)
    cluster_instances = (
        get_worker_instances(ec2_conn, stack_name) +
        [manager_instance, get_master_instance(ec2_conn, stack_name)])
    cluster_hosts = [i.ip_address for i in cluster_instances]

    with cm_tunnel_ctx(manager_instance) as local_port:
        # Connect to CM API
        cm_api = ApiResource('localhost', username='admin', password='admin',
                             server_port=local_port, version=9)
        cloudera_manager = cm_api.get_cloudera_manager()

        # Stop Cloudera Management Service
        print "Stopping Cloudera Management Service"
        mgmt_service = cloudera_manager.get_service()
        mgmt_service.stop().wait()

        # Stop cluster
        print "Stopping the cluster"
        clusters = cm_api.get_all_clusters()
        cluster = clusters.objects[0]
        cluster.stop().wait()

    # Stop all Cloudera Manager Agents
    @parallel
    def stop_cm_agents():
        sudo('service cloudera-scm-agent stop')
    execute(stop_cm_agents, hosts=cluster_hosts)

    # Stop the Cloudera Manager Server
    def stop_cm_server():
        sudo('service cloudera-scm-server stop')
    execute(stop_cm_server, hosts=[manager_instance.ip_address])

    # Cleanup other Java versions and install JDK 1.8
    @parallel
    def swap_jdks():
        sudo('rpm -qa | grep jdk | xargs rpm -e')
        sudo('rm -rf /usr/java/jdk1.6*')
        sudo('rm -rf /usr/java/jdk1.7*')
        run('wget -O jdk-8-linux-x64.rpm --no-cookies --no-check-certificate '
            '--header "Cookie: oraclelicense=accept-securebackup-cookie" '
            'http://download.oracle.com/otn-pub/java/jdk/8u51-b16/'
            'jdk-8u51-linux-x64.rpm')
        sudo('yum install -y jdk-8-linux-x64.rpm')
        append('/home/ec2-user/.bash_profile',
               'export JAVA_HOME=`find /usr/java -name "jdk1.8*"`')
    execute(swap_jdks, hosts=cluster_hosts)

    # Start the Cloudera Manager Server
    def start_cm_server():
        sudo('service cloudera-scm-server start')
    execute(start_cm_server, hosts=[manager_instance.ip_address])

    # Start all Cloudera Manager Agents
    @parallel
    def start_cm_agents():
        sudo('service cloudera-scm-agent start')
    execute(start_cm_agents, hosts=cluster_hosts)

    with cm_tunnel_ctx(manager_instance) as local_port:
        # Connect to CM API
        cm_api = ApiResource('localhost', username='admin', password='admin',
                             server_port=local_port, version=9)
        cloudera_manager = cm_api.get_cloudera_manager()

        # Start the cluster and the mgmt service
        print "Starting the cluster"
        cluster.start().wait()
        print "Starting the Cloudera Management Service"
        cloudera_manager = cm_api.get_cloudera_manager()
        mgmt_service = cloudera_manager.get_service()
        mgmt_service.start().wait()