Python api_client.ApiResource类代码示例

def main():
    """
    Kerberizes a cluster.

    @rtype:   number
    @returns: A number representing the status of success.
    """
    settings = retrieve_args()

    api = ApiResource(settings.host, settings.port, settings.username,
                      settings.password, settings.use_tls, 8)

    cloudera_manager = api.get_cloudera_manager()
    cluster = api.get_cluster(settings.cluster)
    mgmt_service = cloudera_manager.get_service()

    if verify_cloudera_manager_has_kerberos_principal(cloudera_manager):
        wait_for_command('Stopping the cluster', cluster.stop())
        wait_for_command('Stopping MGMT services', mgmt_service.stop())
        configure_services(cluster)
        wait_for_generate_credentials(cloudera_manager)
        wait_for_command('Deploying client configs.', cluster.deploy_client_config())
        wait_for_command('Deploying cluster client configs', cluster.deploy_cluster_client_config())
        wait_for_command('Starting MGMT services', mgmt_service.start())
        wait_for_command('Starting the cluster', cluster.start())
    else:
        print "Cluster does not have Kerberos admin credentials.  Exiting!"

    return 0

def connect(cm_api, cm_username, cm_password, use_proxy=False):
    '''
    Wait for ten minutes for CM to come up
    '''

    for _ in xrange(120):
        try:
            logging.info("Checking CM availability....")
            # change name of proxy if necessary
            proxy = urllib2.ProxyHandler({'http': 'proxy'})

            api = ApiResource(cm_api, username=cm_username, password=cm_password, version=14)

            if use_proxy:
            # pylint: disable=W0212
                api._client._opener.add_handler(proxy)

            cloudera_manager = api.get_cloudera_manager()
            api.get_user(cm_username)

            return api, cloudera_manager
        except Exception:
            logging.warning("CM is not up")
            time.sleep(5)
    logging.error("CM did not come UP")
    sys.exit(-1)

class ImpalaCluster(object):
  def __init__(self, cm_host, cm_cluster_name, username, password):
    self.cm_api = ApiResource(cm_host, username=username, password=password)
    self.hosts = dict()
    self.services = list()
    self.cluster = self.cm_api.get_cluster(cm_cluster_name)
    if self.cluster is None:
      raise RuntimeError, 'Cluster name "%s" not found' % cm_cluster_name

    self.__load_hosts()
    self.__impala_service = ImpalaService(self)

  def _get_all_services(self):
    return self.cluster.get_all_services()

  def get_impala_service(self):
    return self.__impala_service

  def __load_hosts(self):
    self.hosts = dict()
    # Search for all hosts that are in the target cluster.
    # There is no API that provides the list of host in a given cluster, so to find them
    # we must loop through all the hosts and check the cluster name matches.
    for host_info in self.cm_api.get_all_hosts():
      # host_info doesn't include a link to the roleRef so need to do another lookup
      # based on the hostId.
      host = self.cm_api.get_host(host_info.hostId)
      for roleRef.get('clusterName') == self.cluster_name:
        self.hosts[host_info.hostId] = Host(host)
          break

    def __init__(self,
                 hostname=None,
                 port='7180',
                 username='admin',
                 password='admin'):
        '''
        Creates a Resource object that provides API endpoints.

        @param hostname: Hostname of the Cloudera Manager server.
        @param port:     Port of the server. Defaults to 7180 (http).
        @param username: Login name.
        @param password: Login password.
        '''
        config = Config()
        salt   = Salt()

        if hostname is None:
            hostname = salt.function('roles.dict', 'cloudera-cm4-server')['cloudera-cm4-server'][0]

        try:
            logger.debug('Acquiring Cloudera Manager API resource')
            ApiResource.__init__(self, hostname, port, username, password)
        except ApiException:
            logger.error('Problem acquiring Cloudera Manager API resource')
#           sys.exit(1)
        except URLError:
            logger.error('Problem connecting to %s', hostname)
            sys.exit(1)
        else:
            self.config = config
            self.salt   = salt

            cluster = ClouderaManagerCluster(self)
            host    = ClouderaManagerHost(self)

    def __init__(self, hostname=None, port="7180", username="admin", password="admin"):
        """
        Creates a Resource object that provides API endpoints.

        @param hostname: Hostname of the Cloudera Manager server.
        @param port:     Port of the server. Defaults to 7180 (http).
        @param username: Login name.
        @param password: Login password.
        """
        config = Config()
        salt = Salt()

        if hostname is None:
            hostname = salt.function("roles.dict", "cloudera-cm5-server")["cloudera-cm5-server"][0]

        try:
            logger.debug("Acquiring Cloudera Manager API resource")
            ApiResource.__init__(self, hostname, port, username, password)
        except ApiException:
            logger.error("Problem acquiring Cloudera Manager API resource")
        #           sys.exit(1)
        except URLError:
            logger.error("Problem connecting to %s", hostname)
            sys.exit(1)
        else:
            self.config = config
            self.salt = salt

            cluster = ClouderaManagerCluster(self)
            host = ClouderaManagerHost(self)

def adjust_yarn_memory_limits(region, stack_name, restart=True):
    ec2_conn = create_ec2_connection(region)
    manager_instance = get_manager_instance(ec2_conn, stack_name)
    with cm_tunnel_ctx(manager_instance) as local_port:
        cm_api = ApiResource('localhost', username='admin', password='admin',
                             server_port=local_port, version=9)
        cluster = list(cm_api.get_all_clusters())[0]
        host = list(cm_api.get_all_hosts())[0]  # all hosts same instance type
        yarn = filter(lambda x: x.type == 'YARN',
                      list(cluster.get_all_services()))[0]
        rm_cg = filter(lambda x: x.roleType == 'RESOURCEMANAGER',
                       list(yarn.get_all_role_config_groups()))[0]
        nm_cg = filter(lambda x: x.roleType == 'NODEMANAGER',
                       list(yarn.get_all_role_config_groups()))[0]
        rm_cg.update_config({
            'yarn_scheduler_maximum_allocation_mb': (
                int(host.totalPhysMemBytes / 1024. / 1024.)),
            'yarn_scheduler_maximum_allocation_vcores': host.numCores})
        nm_cg.update_config({
            'yarn_nodemanager_resource_memory_mb': (
                int(host.totalPhysMemBytes / 1024. / 1024.)),
            'yarn_nodemanager_resource_cpu_vcores': host.numCores})
        cluster.deploy_client_config().wait()
        if restart:
            cluster.restart().wait()

def list_hosts(host, username, password, cafile):
  context = ssl.create_default_context(cafile=cafile)

  api = ApiResource(host, username=username, password=password, use_tls=True,
                    ssl_context=context)

  for h in api.get_all_hosts():
    print h.hostname

def get_cluster_specs():
    cm_api = ApiResource(os.environ['MANAGER_HOST'], username='admin',
                         password='admin', server_port=7180, version=9)
    host = list(cm_api.get_all_hosts())[0]  # all hosts same instance type
    cluster = list(cm_api.get_all_clusters())[0]
    yarn = filter(lambda x: x.type == 'YARN',
                  list(cluster.get_all_services()))[0]
    return {'num_worker_nodes': len(yarn.get_roles_by_type('NODEMANAGER')),
            'num_cores': host.numCores, 'node_memory': host.totalPhysMemBytes}

def main():
   API = ApiResource(CM_HOST, version=5, username=ADMIN_USER, password=ADMIN_PASS)
   print "Connected to CM host on " + CM_HOST

   CLUSTER = API.get_cluster(CLUSTER_NAME)

   print "About to restart cluster."
   CLUSTER.restart().wait()
   print "Done restarting cluster."

def main():
  parser = cm_args_parser()
  args = parser.parse_args()
  print "connecting to host:" + args.cm_host + "..."
  api = ApiResource(args.cm_host, username=args.cm_user, password=args.cm_password)
  print "host connected, getting cloudera manager "
  MANAGER = api.get_cloudera_manager()
  print "have cloudera manager object" 
  deploy_management(MANAGER, MGMT_SERVICENAME, MGMT_SERVICE_CONFIG, MGMT_ROLE_CONFIG, AMON_ROLENAME, AMON_ROLE_CONFIG, APUB_ROLENAME, APUB_ROLE_CONFIG, ESERV_ROLENAME, ESERV_ROLE_CONFIG, HMON_ROLENAME, HMON_ROLE_CONFIG, SMON_ROLENAME, SMON_ROLE_CONFIG, RMAN_ROLENAME, RMAN_ROLE_CONFIG)
  print "Deployed CM management service " + MGMT_SERVICENAME + " to run on " + CM_HOST

 def getClusterInformation(self):
     api = ApiResource(self.cm_host, username=self.user, password=self.passwd)
     logger.info('Received; user -> %s, password -> %s, host -> %s', self.user, self.passwd, self.cm_host)
     for c in api.get_all_clusters():
         clusterInf = "Cluster name %s and version %s" %(c.name, c.version)
         #print "Cluster name %s and version %s" %(c.name, c.version)
         logger.info("Cluster name %s and version %s", c.name, c.version)
         if c.version == "CDH5":
             cdh5 = c
     return cdh5, clusterInf

def connect_cm(cm_host, cm_username, cm_password):
    """
    Connects to Cloudera Manager API Resource instance to retrieve Endpoint details
    :param cm_host: Cloudera Manager host
    :param cm_username: Username for authentication
    :param cm_password: Password for authentication
    :return:
    """
    api = ApiResource(cm_host, version=6, username=cm_username, password=cm_password)
    cm_manager = api.get_cloudera_manager()
    return api, cm_manager

def find_impala_in_cm(cm_host, cm_user, cm_password, cm_cluster_name):
  """Finds the Impala service in CM and returns an Impala instance."""
  cm = ApiResource(cm_host, username=cm_user, password=cm_password)
  cm_impalas = [service for cluster in cm.get_all_clusters()
                if cm_cluster_name is None or cm_cluster_name == cluster.name
                for service in cluster.get_all_services() if service.type == "IMPALA"]
  if len(cm_impalas) > 1:
    raise Exception("Found %s Impala services in CM;" % len(cm_impalas) +
        " use --cm-cluster-name option to specify which one to use.")
  if len(cm_impalas) == 0:
    raise Exception("No Impala services found in CM")
  return Impala(cm_impalas[0])

 def cm_api_resource(self):
     ar = None
     try:
         ar = ApiResource(self.cm_host, self.cm_port,
                          self.cm_username, self.cm_password)
         ar.echo('Authenticated')  # Issue a sample request to test the conn
     except ApiException, aexc:
         if aexc.code == 401:
             log.debug("Changing default API username to {0}".format(self.cm_username))
             self.cm_username = self.host_username
             self.cm_password = self.host_password
             ar = ApiResource(self.cm_host, self.cm_port,
                              self.cm_username, self.cm_password)
         else:
             log.error("Api Exception connecting to ClouderaManager: {0}".format(aexc))

def main():
    # connect cm api
    api = ApiResource(CM_HOST, 7180, username=CM_USERNAME, password=CM_PASSWORD)
    manager = api.get_cloudera_manager()
    # no need to update cm config
    #manager.update_config(cm_host)
    print("[INFO] Connected to CM host on " + CM_HOST)

    # create cluster object
    try:
        cluster = api.get_cluster(name=CLUSTER_NAME)
    except:
        cluster = init_cluster(api, CLUSTER_NAME, CLUSTER_VERSION, CLUSTER_NODE_COUNT)
    print("[INFO] Initialized cluster " + CLUSTER_NAME + " which uses CDH version " + CLUSTER_VERSION)

    #
    mgmt_servicename = "MGMT"
    amon_role_name = "ACTIVITYMONITOR"
    apub_role_name = "ALERTPUBLISHER"
    eserv_role_name = "EVENTSERVER"
    hmon_role_name = "HOSTMONITOR"
    smon_role_name = "SERVICEMONITOR"
    nav_role_name = "NAVIGATOR"
    navms_role_name = "NAVIGATORMETADATASERVER"
    rman_role_name = "REPORTMANAGER"
    deploy_management(manager, mgmt_servicename, amon_role_name, apub_role_name, eserv_role_name, hmon_role_name, smon_role_name, nav_role_name, navms_role_name, rman_role_name)
    print("[INFO] Deployed CM management service " + mgmt_servicename + " to run on " + CM_HOST)

    #
    assign_roles(api, cluster)
    print("[INFO] all roles have assigned.")

    #
    # Custom role config groups cannot be automatically configured: Gateway Group 1 (error 400)
    try:
        cluster.auto_configure()
    except:
        pass
    update_custom_config(api, cluster)
    print("[INFO] all servies and roles have configured.")
    #
    cmd = cluster.first_run()
    while cmd.success == None:
        cmd = cmd.fetch()
    if not cmd.success:
        print("[ERROR] The first run command failed: " + cmd.resultMessage())
    else:
        print("[INFO] First run successfully executed. Your cluster has been set up!")