Python Log.info方法代码示例

# 需要导入模块: from heron.common.src.python.color import Log [as 别名]
# 或者: from heron.common.src.python.color.Log import info [as 别名]
  def register_watch(self, callback):
    """
    Returns the UUID with which the watch is
    registered. This UUID can be used to unregister
    the watch.
    Returns None if watch could not be registered.

    The argument 'callback' must be a function that takes
    exactly one argument, the topology on which
    the watch was triggered.
    Note that the watch will be unregistered in case
    it raises any Exception the first time.

    This callback is also called at the time
    of registration.
    """
    RETRY_COUNT = 5
    # Retry in case UID is previously
    # generated, just in case...
    for _ in range(RETRY_COUNT):
      # Generate a random UUID.
      uid = uuid.uuid4()
      if uid not in self.watches:
        Log.info("Registering a watch with uid: " + str(uid))
        try:
          callback(self)
        except Exception as e:
          Log.error("Caught exception while triggering callback: " + str(e))
          Log.debug(traceback.format_exc())
          return None
        self.watches[uid] = callback
        return uid
    return None

# 需要导入模块: from heron.common.src.python.color import Log [as 别名]
# 或者: from heron.common.src.python.color.Log import info [as 别名]
def launch_topologies(cl_args, topology_file, tmp_dir):

  # the submitter would have written the .defn file to the tmp_dir
  defn_files = glob.glob(tmp_dir + '/*.defn')

  if len(defn_files) == 0:
    raise Exception("No topologies found")

  try:
    for defn_file in defn_files:

      # load the topology definition from the file
      topology_defn = topology_pb2.Topology()
      try:
        f = open(defn_file, "rb")
        topology_defn.ParseFromString(f.read())
        f.close()

      except:
        raise Exception("Could not open and parse topology defn file %s" % defn_file)

      # launch the topology
      try:
        Log.info("Launching topology \'%s\'" % topology_defn.name)
        launch_a_topology(cl_args, tmp_dir, topology_file, defn_file)
        Log.info("Topology \'%s\' launched successfully" % topology_defn.name)

      except Exception as ex:
        Log.error('Failed to launch topology \'%s\' because %s' % (topology_defn.name, str(ex)))
        raise

  except:
    raise

# 需要导入模块: from heron.common.src.python.color import Log [as 别名]
# 或者: from heron.common.src.python.color.Log import info [as 别名]
def main():
  '''
  Run the command
  :return:
  '''
  # verify if the environment variables are correctly set
  check_environment()

  # create the argument parser
  parser = create_parser()

  # if no argument is provided, print help and exit
  if len(sys.argv[1:]) == 0:
    parser.print_help()
    return 0

  # insert the boolean values for some of the options
  sys.argv = parse.insert_bool_values(sys.argv)

  # parse the args
  args, unknown_args = parser.parse_known_args()
  command_line_args = vars(args)

  try:
    if command_line_args['verbose']:
      opts.set_verbose()
    if command_line_args['trace_execution']:
      opts.set_trace_execution()
  except:
    pass

  # command to be execute
  command = command_line_args['subcommand']

  # file resources to be cleaned when exit
  files = []

  if command != 'help' and command != 'version':
    command_line_args = extract_common_args(command, parser, command_line_args)
    # bail out if args are empty
    if not command_line_args:
      return 1
    # register dirs cleanup function during exit
    files.append(command_line_args['override_config_file'])

  atexit.register(cleanup, files)

  # print the input parameters, if verbose is enabled
  if opts.verbose():
    print command_line_args

  start = time.time()
  retcode = run(command, parser, command_line_args, unknown_args)
  end = time.time()

  if command != 'help':
    sys.stdout.flush()
    Log.info('Elapsed time: %.3fs.' % (end - start))

  return 0 if retcode else 1

# 需要导入模块: from heron.common.src.python.color import Log [as 别名]
# 或者: from heron.common.src.python.color.Log import info [as 别名]
def get_cluster_role_env_topologies(cluster, role, env):
  """Synced API call to get topologies under a cluster submitted by a role under env"""
  instance = tornado.ioloop.IOLoop.instance()
  try:
    return instance.run_sync(lambda: API.get_cluster_role_env_topologies(cluster, role, env))
  except Exception:
    Log.info(traceback.format_exc())
    raise

# 需要导入模块: from heron.common.src.python.color import Log [as 别名]
# 或者: from heron.common.src.python.color.Log import info [as 别名]
 def unregister_watch(self, uid):
   """
   Unregister the watch with the given UUID.
   """
   # Do not raise an error if UUID is
   # not present in the watches.
   Log.info("Unregister a watch with uid: " + str(uid))
   self.watches.pop(uid, None)

# 需要导入模块: from heron.common.src.python.color import Log [as 别名]
# 或者: from heron.common.src.python.color.Log import info [as 别名]
def get_logical_plan(cluster, env, topology, role):
  """Synced API call to get logical plans"""
  instance = tornado.ioloop.IOLoop.instance()
  try:
    return instance.run_sync(lambda: API.get_logical_plan(cluster, env, topology, role))
  except Exception:
    Log.info(traceback.format_exc())
    raise

# 需要导入模块: from heron.common.src.python.color import Log [as 别名]
# 或者: from heron.common.src.python.color.Log import info [as 别名]
def get_topology_metrics(*args):
  """Synced API call to get topology metrics"""
  instance = tornado.ioloop.IOLoop.instance()
  try:
    return instance.run_sync(lambda: API.get_comp_metrics(*args))
  except Exception:
    Log.info(traceback.format_exc())
    raise

# 需要导入模块: from heron.common.src.python.color import Log [as 别名]
# 或者: from heron.common.src.python.color.Log import info [as 别名]
def get_cluster_topologies(cluster):
  """Synced API call to get topologies under a cluster"""
  instance = tornado.ioloop.IOLoop.instance()
  try:
    return instance.run_sync(lambda: API.get_cluster_topologies(cluster))
  except Exception:
    Log.info(traceback.format_exc())
    raise

# 需要导入模块: from heron.common.src.python.color import Log [as 别名]
# 或者: from heron.common.src.python.color.Log import info [as 别名]
def get_clusters():
  """Synced API call to get all cluster names"""
  instance = tornado.ioloop.IOLoop.instance()
  # pylint: disable=unnecessary-lambda
  try:
    return instance.run_sync(lambda: API.get_clusters())
  except Exception:
    Log.info(traceback.format_exc())
    raise

# 需要导入模块: from heron.common.src.python.color import Log [as 别名]
# 或者: from heron.common.src.python.color.Log import info [as 别名]
def get_component_metrics(component, cluster, env, topology, role):
  """Synced API call to get component metrics"""
  all_queries = metric_queries()
  try:
    result = get_topology_metrics(cluster, env, topology, component, [],
                                  all_queries, [0, -1], role)
    return result["metrics"]
  except Exception:
    Log.info(traceback.format_exc())
    raise

# 需要导入模块: from heron.common.src.python.color import Log [as 别名]
# 或者: from heron.common.src.python.color.Log import info [as 别名]
def parse_cluster_role_env(cluster_role_env, config_path):
  """Parse cluster/[role]/[environ], supply default, if not provided, not required"""
  parts = cluster_role_env.split('/')[:3]
  Log.info("Using config file under %s" % config_path)
  if not os.path.isdir(config_path):
    Log.error("Config path cluster directory does not exist: %s" % config_path)
    raise Exception("Invalid config path")

  # if cluster/role/env is not completely provided, check further
  if len(parts) < 3:

    cli_conf_file = os.path.join(config_path, CLIENT_YAML)

    # if client conf doesn't exist, use default value
    if not os.path.isfile(cli_conf_file):
      if len(parts) == 1:
        parts.append(getpass.getuser())
      if len(parts) == 2:
        parts.append(ENVIRON)
    else:
      cli_confs = {}
      with open(cli_conf_file, 'r') as conf_file:
        tmp_confs = yaml.load(conf_file)
        # the return value of yaml.load can be None if conf_file is an empty file
        if tmp_confs is not None:
          cli_confs = tmp_confs
        else:
          print "Failed to read: %s due to it is empty" % (CLIENT_YAML)

      # if role is required but not provided, raise exception
      if len(parts) == 1:
        if (IS_ROLE_REQUIRED in cli_confs) and (cli_confs[IS_ROLE_REQUIRED] is True):
          raise Exception(
              "role required but not provided (cluster/role/env = %s). See %s in %s" %
              (cluster_role_env, IS_ROLE_REQUIRED, CLIENT_YAML))
        else:
          parts.append(getpass.getuser())

      # if environ is required but not provided, raise exception
      if len(parts) == 2:
        if (IS_ENV_REQUIRED in cli_confs) and (cli_confs[IS_ENV_REQUIRED] is True):
          raise Exception(
              "environ required but not provided (cluster/role/env = %s). See %s in %s" %
              (cluster_role_env, IS_ENV_REQUIRED, CLIENT_YAML))
        else:
          parts.append(ENVIRON)

  # if cluster or role or environ is empty, print
  if len(parts[0]) == 0 or len(parts[1]) == 0 or len(parts[2]) == 0:
    print "Failed to parse"
    sys.exit(1)

  return (parts[0], parts[1], parts[2])

# 需要导入模块: from heron.common.src.python.color import Log [as 别名]
# 或者: from heron.common.src.python.color.Log import info [as 别名]
  def setTopologyInfo(self, topology):
    """
    Extracts info from the stored proto states and
    convert it into representation that is exposed using
    the API.
    This method is called on any change for the topology.
    For example, when a container moves and its host or some
    port changes. All the information is parsed all over
    again and cache is updated.
    """
    # Execution state is the most basic info.
    # If there is no execution state, just return
    # as the rest of the things don't matter.
    if not topology.execution_state:
      Log.info("No execution state found for: " + topology.name)
      return

    Log.info("Setting topology info for topology: " + topology.name)
    has_physical_plan = True
    if not topology.physical_plan:
      has_physical_plan = False

    has_tmaster_location = True
    if not topology.tmaster:
      has_tmaster_location = False

    has_scheduler_location = True
    if not topology.scheduler_location:
      has_scheduler_location = False

    top = {
        "name": topology.name,
        "id": topology.id,
        "logical_plan": None,
        "physical_plan": None,
        "execution_state": None,
        "tmaster_location": None,
        "scheduler_location": None,
    }

    executionState = self.extract_execution_state(topology)
    executionState["has_physical_plan"] = has_physical_plan
    executionState["has_tmaster_location"] = has_tmaster_location
    executionState["has_scheduler_location"] = has_scheduler_location

    top["execution_state"] = executionState
    top["logical_plan"] = self.extract_logical_plan(topology)
    top["physical_plan"] = self.extract_physical_plan(topology)
    top["tmaster_location"] = self.extract_tmaster(topology)
    top["scheduler_location"] = self.extract_scheduler_location(topology)

    self.topologyInfos[(topology.name, topology.state_manager_name)] = top

# 需要导入模块: from heron.common.src.python.color import Log [as 别名]
# 或者: from heron.common.src.python.color.Log import info [as 别名]
    def on_topologies_watch(state_manager, topologies):
      """watch topologies"""
      Log.info("State watch triggered for topologies.")
      Log.debug("Topologies: " + str(topologies))
      existingTopologies = self.getTopologiesForStateLocation(state_manager.name)
      existingTopNames = map(lambda t: t.name, existingTopologies)
      Log.debug("Existing topologies: " + str(existingTopNames))
      for name in existingTopNames:
        if name not in topologies:
          Log.info("Removing topology: %s in rootpath: %s",
                   name, state_manager.rootpath)
          self.removeTopology(name, state_manager.name)

      for name in topologies:
        if name not in existingTopNames:
          self.addNewTopology(state_manager, name)

# 需要导入模块: from heron.common.src.python.color import Log [as 别名]
# 或者: from heron.common.src.python.color.Log import info [as 别名]
  def addNewTopology(self, state_manager, topologyName):
    """
    Adds a topology in the local cache, and sets a watch
    on any changes on the topology.
    """
    topology = Topology(topologyName, state_manager.name)
    Log.info("Adding new topology: %s, state_manager: %s",
             topologyName, state_manager.name)
    self.topologies.append(topology)

    # Register a watch on topology and change
    # the topologyInfo on any new change.
    topology.register_watch(self.setTopologyInfo)

    def on_topology_pplan(data):
      """watch physical plan"""
      Log.info("Watch triggered for topology pplan: " + topologyName)
      topology.set_physical_plan(data)
      if not data:
        Log.debug("No data to be set")

    def on_topology_execution_state(data):
      """watch execution state"""
      Log.info("Watch triggered for topology execution state: " + topologyName)
      topology.set_execution_state(data)
      if not data:
        Log.debug("No data to be set")

    def on_topology_tmaster(data):
      """set tmaster"""
      Log.info("Watch triggered for topology tmaster: " + topologyName)
      topology.set_tmaster(data)
      if not data:
        Log.debug("No data to be set")

    def on_topology_scheduler_location(data):
      """set scheduler location"""
      Log.info("Watch triggered for topology scheduler location: " + topologyName)
      topology.set_scheduler_location(data)
      if not data:
        Log.debug("No data to be set")

    # Set watches on the pplan, execution_state, tmaster and scheduler_location.
    state_manager.get_pplan(topologyName, on_topology_pplan)
    state_manager.get_execution_state(topologyName, on_topology_execution_state)
    state_manager.get_tmaster(topologyName, on_topology_tmaster)
    state_manager.get_scheduler_location(topologyName, on_topology_scheduler_location)

# 需要导入模块: from heron.common.src.python.color import Log [as 别名]
# 或者: from heron.common.src.python.color.Log import info [as 别名]
def run(command, parser, cl_args, unknown_args, action):
  '''
  helper function to take action on topologies
  :param command:
  :param parser:
  :param cl_args:
  :param unknown_args:
  :param action:        description of action taken
  :return:
  '''
  try:
    topology_name = cl_args['topology-name']

    new_args = [
        "--cluster", cl_args['cluster'],
        "--role", cl_args['role'],
        "--environment", cl_args['environ'],
        "--heron_home", utils.get_heron_dir(),
        "--config_path", cl_args['config_path'],
        "--override_config_file", cl_args['override_config_file'],
        "--release_file", utils.get_heron_release_file(),
        "--topology_name", topology_name,
        "--command", command,
    ]

    if opts.verbose():
      new_args.append("--verbose")

    lib_jars = utils.get_heron_libs(jars.scheduler_jars() + jars.statemgr_jars())

    # invoke the runtime manager to kill the topology
    execute.heron_class(
        'com.twitter.heron.scheduler.RuntimeManagerMain',
        lib_jars,
        extra_jars=[],
        args=new_args
    )

  except Exception:
    Log.error('Failed to %s \'%s\'' % (action, topology_name))
    return False

  Log.info('Successfully %s \'%s\'' % (action, topology_name))
  return True