Python Logger.warning方法代码示例

# 需要导入模块: from resource_management.core.logger import Logger [as 别名]
# 或者: from resource_management.core.logger.Logger import warning [as 别名]
def bootstrap_standby_namenode(params, use_path=False):

  bin_path = os.path.join(params.hadoop_bin_dir, '') if use_path else ""

  try:
    iterations = 50
    bootstrap_cmd = format("{bin_path}hdfs namenode -bootstrapStandby -nonInteractive")
    # Blue print based deployments start both NN in parallel and occasionally
    # the first attempt to bootstrap may fail. Depending on how it fails the
    # second attempt may not succeed (e.g. it may find the folder and decide that
    # bootstrap succeeded). The solution is to call with -force option but only
    # during initial start
    if params.command_phase == "INITIAL_START":
      bootstrap_cmd = format("{bin_path}hdfs namenode -bootstrapStandby -nonInteractive -force")
    Logger.info("Boostrapping standby namenode: %s" % (bootstrap_cmd))
    for i in range(iterations):
      Logger.info('Try %d out of %d' % (i+1, iterations))
      code, out = shell.call(bootstrap_cmd, logoutput=False, user=params.hdfs_user)
      if code == 0:
        Logger.info("Standby namenode bootstrapped successfully")
        return True
      elif code == 5:
        Logger.info("Standby namenode already bootstrapped")
        return True
      else:
        Logger.warning('Bootstrap standby namenode failed with %d error code. Will retry' % (code))
  except Exception as ex:
    Logger.error('Bootstrap standby namenode threw an exception. Reason %s' %(str(ex)))
  return False

# 需要导入模块: from resource_management.core.logger import Logger [as 别名]
# 或者: from resource_management.core.logger.Logger import warning [as 别名]
def get_current_version(use_upgrading_version_during_upgrade=True):
  """
  Get the effective version to use to copy the tarballs to.
  :param use_upgrading_version_during_upgrade: True, except when the RU/EU hasn't started yet.
  :return: Version, or False if an error occurred.
  """
  upgrade_direction = default("/commandParams/upgrade_direction", None)
  is_stack_upgrade = upgrade_direction is not None
  current_version = default("/hostLevelParams/current_version", None)
  Logger.info("Default version is {0}".format(current_version))
  if is_stack_upgrade:
    if use_upgrading_version_during_upgrade:
      # This is the version going to. In the case of a downgrade, it is the lower version.
      current_version = default("/commandParams/version", None)
      Logger.info("Because this is a Stack Upgrade, will use version {0}".format(current_version))
    else:
      Logger.info("This is a Stack Upgrade, but keep the version unchanged.")
  else:
    if current_version is None:
      # During normal operation, the first installation of services won't yet know about the version, so must rely
      # on <stack-selector> to get it.
      stack_version = _get_single_version_from_stack_select()
      if stack_version:
        Logger.info("Will use stack version {0}".format(stack_version))
        current_version = stack_version

  if current_version is None:
    message_suffix = "during stack %s" % str(upgrade_direction) if is_stack_upgrade else ""
    Logger.warning("Cannot copy tarball because unable to determine current version {0}.".format(message_suffix))
    return False

  return current_version

# 需要导入模块: from resource_management.core.logger import Logger [as 别名]
# 或者: from resource_management.core.logger.Logger import warning [as 别名]
def select(stack_name, package, version, try_create=True, ignore_errors=False):
  """
  Selects a config version for the specified package. If this detects that
  the stack supports configuration versioning but /etc/<component>/conf is a
  directory, then it will attempt to bootstrap the conf.backup directory and change
  /etc/<component>/conf into a symlink.

  :param stack_name: the name of the stack
  :param package: the name of the package, as-used by <conf-selector-tool>
  :param version: the version number to create
  :param try_create: optional argument to attempt to create the directory before setting it
  :param ignore_errors: optional argument to ignore any error and simply log a warning
  """
  try:
    # do nothing if the stack does not support versioned configurations
    if not _valid(stack_name, package, version):
      return

    if try_create:
      create(stack_name, package, version)

    shell.checked_call(_get_cmd("set-conf-dir", package, version), logoutput=False, quiet=False, sudo=True)

    # for consistency sake, we must ensure that the /etc/<component>/conf symlink exists and
    # points to <stack-root>/current/<component>/conf - this is because some people still prefer to
    # use /etc/<component>/conf even though <stack-root> is the "future"
    package_dirs = get_package_dirs()
    if package in package_dirs:
      Logger.info("Ensuring that {0} has the correct symlink structure".format(package))

      directory_list = package_dirs[package]
      for directory_structure in directory_list:
        conf_dir = directory_structure["conf_dir"]
        current_dir = directory_structure["current_dir"]

        # if /etc/<component>/conf is missing or is not a symlink
        if not os.path.islink(conf_dir):
          # if /etc/<component>/conf is not a link and it exists, convert it to a symlink
          if os.path.exists(conf_dir):
            parent_directory = os.path.dirname(conf_dir)
            conf_backup_dir = os.path.join(parent_directory, "conf.backup")

            # create conf.backup and copy files to it (if it doesn't exist)
            Execute(("cp", "-R", "-p", conf_dir, conf_backup_dir),
              not_if = format("test -e {conf_backup_dir}"), sudo = True)

            # delete the old /etc/<component>/conf directory and link to the backup
            Directory(conf_dir, action="delete")
            Link(conf_dir, to = conf_backup_dir)
          else:
            # missing entirely
            # /etc/<component>/conf -> <stack-root>/current/<component>/conf
            Link(conf_dir, to = current_dir)

  except Exception, exception:
    if ignore_errors is True:
      Logger.warning("Could not select the directory for package {0}. Error: {1}".format(package,
        str(exception)))
    else:
      raise

# 需要导入模块: from resource_management.core.logger import Logger [as 别名]
# 或者: from resource_management.core.logger.Logger import warning [as 别名]
    def service_check(self, env):
        import params
        env.set_params(params)

        if not os.path.isfile(params.solr_config_pid_file):
            Logger.error(format("PID file {solr_config_pid_file} does not exist"))
            exit(1)

        if not params.solr_collection_sample_create:
            Logger.info("Create sample collection unchecked, skipping ...")
            return

        if exists_collection(params.solr_collection_name):
            Logger.warning(format("Collection {solr_collection_name} already exists, skipping ..."))
            return

        if not params.solr_cloud_mode:
            Execute(
                    format(
                            '{solr_config_bin_dir}/solr create_core -c {solr_collection_name}' +
                            ' -d {solr_collection_config_dir} -p {solr_config_port} >> {solr_config_service_log_file} 2>&1'
                    ),
                    environment={'JAVA_HOME': params.java64_home},
                    user=params.solr_config_user
            )
        else:
            Execute(format(
                    '{solr_config_bin_dir}/solr create_collection -c {solr_collection_name}' +
                    ' -d {solr_collection_config_dir} -p {solr_config_port}' +
                    ' -s {solr_collection_shards} -rf {solr_collection_replicas}' +
                    ' >> {solr_config_service_log_file} 2>&1'),
                    environment={'JAVA_HOME': params.java64_home},
                    user=params.solr_config_user
            )

# 需要导入模块: from resource_management.core.logger import Logger [as 别名]
# 或者: from resource_management.core.logger.Logger import warning [as 别名]
def refresh_configs(params):
  if not is_zk_configured(params):
    Logger.warning("The expected flag file '" + params.zk_configured_flag_file + "'indicating that Zookeeper has been configured does not exist. Skipping patching. An administrator should look into this.")
    return
  check_indexer_parameters()
  patch_global_config(params)
  pull_config(params)

# 需要导入模块: from resource_management.core.logger import Logger [as 别名]
# 或者: from resource_management.core.logger.Logger import warning [as 别名]
 def get_hdp_version():
   if not options.hdp_version:
     # Ubuntu returns: "stdin: is not a tty", as subprocess output.
     tmpfile = tempfile.NamedTemporaryFile()
     out = None
     with open(tmpfile.name, 'r+') as file:
       get_hdp_version_cmd = '/usr/bin/hdp-select status %s > %s' % ('hadoop-mapreduce-historyserver', tmpfile.name)
       code, stdoutdata = shell.call(get_hdp_version_cmd)
       out = file.read()
     pass
     if code != 0 or out is None:
       Logger.warning("Could not verify HDP version by calling '%s'. Return Code: %s, Output: %s." %
                      (get_hdp_version_cmd, str(code), str(out)))
       return 1
    
     matches = re.findall(r"([\d\.]+\-\d+)", out)
     hdp_version = matches[0] if matches and len(matches) > 0 else None
    
     if not hdp_version:
       Logger.error("Could not parse HDP version from output of hdp-select: %s" % str(out))
       return 1
   else:
     hdp_version = options.hdp_version
     
   return hdp_version

# 需要导入模块: from resource_management.core.logger import Logger [as 别名]
# 或者: from resource_management.core.logger.Logger import warning [as 别名]
def get_role_component_current_hdp_version():
  """
  Gets the current HDP version of the component that this role command is for.
  :return:  the current HDP version of the specified component or None
  """
  hdp_select_component = None
  role = default("/role", "")
  role_command =  default("/roleCommand", "")

  if role in SERVER_ROLE_DIRECTORY_MAP:
    hdp_select_component = SERVER_ROLE_DIRECTORY_MAP[role]
  elif role_command == "SERVICE_CHECK" and role in SERVICE_CHECK_DIRECTORY_MAP:
    hdp_select_component = SERVICE_CHECK_DIRECTORY_MAP[role]

  if hdp_select_component is None:
    return None

  current_hdp_version = get_hdp_version(hdp_select_component)

  if current_hdp_version is None:
    Logger.warning("Unable to determine hdp-select version for {0}".format(
      hdp_select_component))
  else:
    Logger.info("{0} is currently at version {1}".format(
      hdp_select_component, current_hdp_version))

  return current_hdp_version

# 需要导入模块: from resource_management.core.logger import Logger [as 别名]
# 或者: from resource_management.core.logger.Logger import warning [as 别名]
    def solr_schema_install(self, env):
        from params import params
        env.set_params(params)
        Logger.info("Installing Solr schemas")
        if self.__params.security_enabled:
            metron_security.kinit(self.__params.kinit_path_local,
                                  self.__params.solr_keytab_path,
                                  self.__params.solr_principal_name,
                                  self.__params.solr_user)

        try:
            commands = IndexingCommands(params)
            for collection_name in commands.get_solr_schemas():
                # install the schema
                cmd = format((
                "export ZOOKEEPER={solr_zookeeper_url};"
                "export SECURITY_ENABLED={security_enabled};"
                ))
                cmd += "{0}/bin/create_collection.sh {1};"

                Execute(
                cmd.format(params.metron_home, collection_name),
                user=self.__params.solr_user)
            return True

        except Exception as e:
            msg = "WARNING: Solr schemas could not be installed.  " \
                  "Is Solr running?  Will reattempt install on next start.  error={0}"
            Logger.warning(msg.format(e))
            return False

# 需要导入模块: from resource_management.core.logger import Logger [as 别名]
# 或者: from resource_management.core.logger.Logger import warning [as 别名]
def get_role_component_current_stack_version():
  """
  Gets the current HDP version of the component that this role command is for.
  :return:  the current HDP version of the specified component or None
  """
  stack_select_component = None
  role = default("/role", "")
  role_command =  default("/roleCommand", "")
  stack_selector_name = stack_tools.get_stack_tool_name(stack_tools.STACK_SELECTOR_NAME)

  if role in SERVER_ROLE_DIRECTORY_MAP:
    stack_select_component = SERVER_ROLE_DIRECTORY_MAP[role]
  elif role_command == "SERVICE_CHECK" and role in SERVICE_CHECK_DIRECTORY_MAP:
    stack_select_component = SERVICE_CHECK_DIRECTORY_MAP[role]

  if stack_select_component is None:
    return None

  current_stack_version = get_stack_version(stack_select_component)

  if current_stack_version is None:
    Logger.warning("Unable to determine {0} version for {1}".format(
      stack_selector_name, stack_select_component))
  else:
    Logger.info("{0} is currently at version {1}".format(
      stack_select_component, current_stack_version))

  return current_stack_version

# 需要导入模块: from resource_management.core.logger import Logger [as 别名]
# 或者: from resource_management.core.logger.Logger import warning [as 别名]
def _get_tar_source_and_dest_folder(tarball_prefix):
  """
  :param tarball_prefix: Prefix of the tarball must be one of tez, hive, mr, pig
  :return: Returns a tuple of (x, y) after verifying the properties
  """
  component_tar_source_file = default("/configurations/cluster-env/%s%s" % (tarball_prefix.lower(), TAR_SOURCE_SUFFIX), None)
  # E.g., /usr/hdp/current/hadoop-client/tez-{{ hdp_stack_version }}.tar.gz

  component_tar_destination_folder = default("/configurations/cluster-env/%s%s" % (tarball_prefix.lower(), TAR_DESTINATION_FOLDER_SUFFIX), None)
  # E.g., hdfs:///hdp/apps/{{ hdp_stack_version }}/mapreduce/

  if not component_tar_source_file or not component_tar_destination_folder:
    Logger.warning("Did not find %s tar source file and destination folder properties in cluster-env.xml" %
                   tarball_prefix)
    return None, None

  if component_tar_source_file.find("/") == -1:
    Logger.warning("The tar file path %s is not valid" % str(component_tar_source_file))
    return None, None

  if not component_tar_destination_folder.endswith("/"):
    component_tar_destination_folder = component_tar_destination_folder + "/"

  if not component_tar_destination_folder.startswith("hdfs://"):
    return None, None

  return component_tar_source_file, component_tar_destination_folder

# 需要导入模块: from resource_management.core.logger import Logger [as 别名]
# 或者: from resource_management.core.logger.Logger import warning [as 别名]
def link_component_conf_to_versioned_config(package, version):
  """
  Make /usr/hdp/[version]/[component]/conf point to the versioned config.
  """
  try:
    select("HDP", package, version)
  except Exception, e:
    Logger.warning("Could not select the directory for package {0}. Error: {1}".format(package, e))

# 需要导入模块: from resource_management.core.logger import Logger [as 别名]
# 或者: from resource_management.core.logger.Logger import warning [as 别名]
 def status(self, env):
     import status_params
     env.set_params(status_params)
     # warring defalut port is 9090
     process_check_command = "sudo ps aux | grep '/usr/hdp/2.2.0.0-2041/webide/webide-app'  | grep -v 'grep' "
     output = self.command_exe(process_check_command)
     if not output:
         Logger.warning("{0} did not started!".format("webide APP server"))
         raise ComponentIsNotRunning()

# 需要导入模块: from resource_management.core.logger import Logger [as 别名]
# 或者: from resource_management.core.logger.Logger import warning [as 别名]
def copy_tarballs_to_hdfs(source, dest, hdp_select_component_name, component_user, file_owner, group_owner):
  """
  :param tarball_prefix: Prefix of the tarball must be one of tez, hive, mr, pig
  :param hdp_select_component_name: Component name to get the status to determine the version
  :param component_user: User that will execute the Hadoop commands
  :param file_owner: Owner of the files copied to HDFS (typically hdfs account)
  :param group_owner: Group owner of the files copied to HDFS (typically hadoop group)
  :return: Returns 0 on success, 1 if no files were copied, and in some cases may raise an exception.
 
  In order to call this function, params.py must have all of the following,
  hdp_stack_version, kinit_path_local, security_enabled, hdfs_user, hdfs_principal_name, hdfs_user_keytab,
  hadoop_bin_dir, hadoop_conf_dir, and HdfsDirectory as a partial function.
  """
 
  component_tar_source_file, component_tar_destination_folder = source, dest
 
  if not os.path.exists(component_tar_source_file):
    Logger.warning("Could not find file: %s" % str(component_tar_source_file))
    return 1
 
  # Ubuntu returns: "stdin: is not a tty", as subprocess output.
  tmpfile = tempfile.NamedTemporaryFile()
  out = None
  with open(tmpfile.name, 'r+') as file:
    get_hdp_version_cmd = '/usr/bin/hdp-select status %s > %s' % (hdp_select_component_name, tmpfile.name)
    code, stdoutdata = shell.call(get_hdp_version_cmd)
    out = file.read()
  pass
  if code != 0 or out is None:
    Logger.warning("Could not verify HDP version by calling '%s'. Return Code: %s, Output: %s." %
                   (get_hdp_version_cmd, str(code), str(out)))
    return 1
 
  matches = re.findall(r"([\d\.]+\-\d+)", out)
  hdp_version = matches[0] if matches and len(matches) > 0 else None
 
  if not hdp_version:
    Logger.error("Could not parse HDP version from output of hdp-select: %s" % str(out))
    return 1
 
  file_name = os.path.basename(component_tar_source_file)
  destination_file = os.path.join(component_tar_destination_folder, file_name)
  destination_file = destination_file.replace("{{ hdp_stack_version }}", hdp_version)
 

  kinit_if_needed = ""
  if params.security_enabled:
    kinit_if_needed = format("{kinit_path_local} -kt {hdfs_user_keytab} {hdfs_principal_name};")
 
  if kinit_if_needed:
    Execute(kinit_if_needed,
            user=component_user,
            path='/bin'
    )
 
  source_and_dest_pairs = [(component_tar_source_file, destination_file), ]
  return _copy_files(source_and_dest_pairs, file_owner, group_owner, kinit_if_needed)

# 需要导入模块: from resource_management.core.logger import Logger [as 别名]
# 或者: from resource_management.core.logger.Logger import warning [as 别名]
 def status(self, env):
     import status_params
     env.set_params(status_params)
     
     process_check_command = "ps -ef | grep 'org.apache.spark.sql.hive.thriftserver.HiveThriftServer2' | grep -v grep"
     output = self.command_exe(process_check_command)
     if not output:
         Logger.warning("{0} did not started!".format("Spark livy server"))
         raise ComponentIsNotRunning()

# 需要导入模块: from resource_management.core.logger import Logger [as 别名]
# 或者: from resource_management.core.logger.Logger import warning [as 别名]
    def status(self, env):
        import status_params

        env.set_params(status_params)

        process_check_command = "ps -ef | grep 'com.cloudera.hue.livy.server.Main' | grep -v grep"
        output = self.command_exe(process_check_command)
        if not output:
            Logger.warning("{0} did not started!".format("Spark livy server"))
            raise ComponentIsNotRunning()