Python log.Log类代码示例

  def create_pplan(self, topologyName, pplan):
    """ create physical plan """
    if not pplan or not pplan.IsInitialized():
      raise StateException("Physical Plan protobuf not init properly",
                           StateException.EX_TYPE_PROTOBUF_ERROR), None, sys.exc_info()[2]

    path = self.get_pplan_path(topologyName)
    LOG.info("Adding topology: {0} to path: {1}".format(
        topologyName, path))
    pplanString = pplan.SerializeToString()
    try:
      self.client.create(path, value=pplanString, makepath=True)
      return True
    except NoNodeError:
      raise StateException("NoNodeError while creating pplan",
                           StateException.EX_TYPE_NO_NODE_ERROR), None, sys.exc_info()[2]
    except NodeExistsError:
      raise StateException("NodeExistsError while creating pplan",
                           StateException.EX_TYPE_NODE_EXISTS_ERROR), None, sys.exc_info()[2]
    except ZookeeperError:
      raise StateException("Zookeeper while creating pplan",
                           StateException.EX_TYPE_ZOOKEEPER_ERROR), None, sys.exc_info()[2]
    except Exception:
      # Just re raise the exception.
      raise

def get_all_zk_state_managers(conf):
  """
  Connects to all the zookeeper state_managers and returns
  the connected state_managers instances.
  """
  state_managers = []
  state_locations = conf.get_state_locations_of_type("zookeeper")
  for location in state_locations:
    name = location['name']
    hostport = location['hostport']
    host = None
    port = None
    if ':' in hostport:
      hostportlist = hostport.split(':')
      if len(hostportlist) == 2:
        host = hostportlist[0]
        port = int(hostportlist[1])
    if not host or not port:
      raise Exception("Hostport for %s must be of the format 'host:port'." % (name))
    tunnelhost = location['tunnelhost']
    rootpath = location['rootpath']
    LOG.info("Connecting to zk hostport: " + host + ":" + str(port) + " rootpath: " + rootpath)
    state_manager = ZkStateManager(name, host, port, rootpath, tunnelhost)
    try:
      state_manager.start()
    except Exception as e:
      LOG.error("Exception while connecting to state_manager.")
      traceback.print_exc()
    state_managers.append(state_manager)

  return state_managers

def get_all_zk_state_managers(conf):
  """
  Creates all the zookeeper state_managers and returns
  them in a list
  """
  state_managers = []
  state_locations = conf.get_state_locations_of_type("zookeeper")
  for location in state_locations:
    name = location['name']
    hostport = location['hostport']
    hostportlist = []
    for hostportpair in hostport.split(','):
      host = None
      port = None
      if ':' in hostport:
        hostandport = hostportpair.split(':')
        if len(hostandport) == 2:
          host = hostandport[0]
          port = int(hostandport[1])
      if not host or not port:
        raise Exception("Hostport for %s must be of the format 'host:port'." % (name))
      hostportlist.append((host, port))
    tunnelhost = location['tunnelhost']
    rootpath = location['rootpath']
    LOG.info("Connecting to zk hostports: " + str(hostportlist) + " rootpath: " + rootpath)
    state_manager = ZkStateManager(name, hostportlist, rootpath, tunnelhost)
    state_managers.append(state_manager)

  return state_managers

  def _get_execution_state_with_watch(self, topologyName, callback, isWatching):
    """
    Helper function to get execution state with
    a callback. The future watch is placed
    only if isWatching is True.
    """
    path = self.get_execution_state_path(topologyName)
    if isWatching:
      LOG.info("Adding data watch for path: " + path)

    # pylint: disable=unused-variable, unused-argument
    @self.client.DataWatch(path)
    def watch_execution_state(data, stats):
      """ invoke callback to watch execute state """
      if data:
        executionState = ExecutionState()
        executionState.ParseFromString(data)
        callback(executionState)
      else:
        callback(None)

      # Returning False will result in no future watches
      # being triggered. If isWatching is True, then
      # the future watches will be triggered.
      return isWatching

  def create_execution_state(self, topologyName, executionState):
    """ create execution state """
    if not executionState or not executionState.IsInitialized():
      raise StateException("Execution State protobuf not init properly",
                           StateException.EX_TYPE_PROTOBUF_ERROR), None, sys.exc_info()[2]

    path = self.get_execution_state_path(topologyName)
    LOG.info("Adding topology: {0} to path: {1}".format(
        topologyName, path))
    executionStateString = executionState.SerializeToString()
    try:
      self.client.create(path, value=executionStateString, makepath=True)
      return True
    except NoNodeError:
      raise StateException("NoNodeError while creating execution state",
                           StateException.EX_TYPE_NO_NODE_ERROR), None, sys.exc_info()[2]
    except NodeExistsError:
      raise StateException("NodeExistsError while creating execution state",
                           StateException.EX_TYPE_NODE_EXISTS_ERROR), None, sys.exc_info()[2]
    except ZookeeperError:
      raise StateException("Zookeeper while creating execution state",
                           StateException.EX_TYPE_ZOOKEEPER_ERROR), None, sys.exc_info()[2]
    except Exception:
      # Just re raise the exception.
      raise

  def _get_scheduler_location_with_watch(self, topologyName, callback, isWatching):
    """
    Helper function to get scheduler location with
    a callback. The future watch is placed
    only if isWatching is True.
    """
    path = self.get_scheduler_location_path(topologyName)
    if isWatching:
      LOG.info("Adding data watch for path: " + path)

    # pylint: disable=unused-variable, unused-argument
    @self.client.DataWatch(path)
    def watch_scheduler_location(data, stats):
      """ invoke callback to watch scheduler location """
      if data:
        scheduler_location = SchedulerLocation()
        scheduler_location.ParseFromString(data)
        callback(scheduler_location)
      else:
        callback(None)

      # Returning False will result in no future watches
      # being triggered. If isWatching is True, then
      # the future watches will be triggered.
      return isWatching

  def _get_topology_with_watch(self, topologyName, callback, isWatching):
    """
    Helper function to get pplan with
    a callback. The future watch is placed
    only if isWatching is True.
    """
    path = self.get_topology_path(topologyName)
    if isWatching:
      LOG.info("Adding data watch for path: " + path)

    # pylint: disable=unused-variable, unused-argument
    @self.client.DataWatch(path)
    def watch_topology(data, stats):
      """ watch topology """
      if data:
        topology = Topology()
        topology.ParseFromString(data)
        callback(topology)
      else:
        callback(None)

      # Returning False will result in no future watches
      # being triggered. If isWatching is True, then
      # the future watches will be triggered.
      return isWatching

  def create_topology(self, topologyName, topology):
    """ crate topology """
    if not topology or not topology.IsInitialized():
      raise_(StateException("Topology protobuf not init properly",
                            StateException.EX_TYPE_PROTOBUF_ERROR), sys.exc_info()[2])

    path = self.get_topology_path(topologyName)
    LOG.info("Adding topology: {0} to path: {1}".format(
        topologyName, path))
    topologyString = topology.SerializeToString()
    try:
      self.client.create(path, value=topologyString, makepath=True)
      return True
    except NoNodeError:
      raise_(StateException("NoNodeError while creating topology",
                            StateException.EX_TYPE_NO_NODE_ERROR), sys.exc_info()[2])
    except NodeExistsError:
      raise_(StateException("NodeExistsError while creating topology",
                            StateException.EX_TYPE_NODE_EXISTS_ERROR), sys.exc_info()[2])
    except ZookeeperError:
      raise_(StateException("Zookeeper while creating topology",
                            StateException.EX_TYPE_ZOOKEEPER_ERROR), sys.exc_info()[2])
    except Exception:
      # Just re raise the exception.
      raise

def get_all_state_managers(conf):
  """
  @param conf - An instance of Config class
  Reads the config for requested state managers.
  Instantiates them, start and then return them.
  """
  state_managers = []
  try:
    state_managers.extend(get_all_zk_state_managers(conf))
    state_managers.extend(get_all_file_state_managers(conf))
    return state_managers
  except Exception as ex:
    LOG.error("Exception while getting state_managers.")
    raise ex

 def is_host_port_reachable(self):
   """
   Returns true if the host is reachable. In some cases, it may not be reachable a tunnel
   must be used.
   """
   for hostport in self.hostportlist:
     try:
       socket.create_connection(hostport, StateManager.TIMEOUT_SECONDS)
       return True
     except:
       LOG.info("StateManager %s Unable to connect to host: %s port %i"
                % (self.name, hostport[0], hostport[1]))
       continue
   return False

def get_all_file_state_managers(conf):
  """
  Returns all the file state_managers.
  """
  state_managers = []
  state_locations = conf.get_state_locations_of_type("file")
  for location in state_locations:
    name = location['name']
    rootpath = os.path.expanduser(location['rootpath'])
    LOG.info("Connecting to file state with rootpath: " + rootpath)
    state_manager = FileStateManager(name, rootpath)
    state_managers.append(state_manager)

  return state_managers

  def _get_topologies_with_watch(self, callback, isWatching):
    """
    Helper function to get topologies with
    a callback. The future watch is placed
    only if isWatching is True.
    """
    path = self.get_topologies_path()
    if isWatching:
      LOG.info("Adding children watch for path: " + path)

    @self.client.ChildrenWatch(path)
    def watch_topologies(topologies):
      callback(topologies)

      # Returning False will result in no future watches
      # being triggered. If isWatching is True, then
      # the future watches will be triggered.
      return isWatching

 def delete_execution_state(self, topologyName):
   path = self.get_execution_state_path(topologyName)
   LOG.info("Removing topology: {0} from path: {1}".format(
     topologyName, path))
   try:
     self.client.delete(path)
     return True
   except NoNodeError as e:
     raise StateException("NoNodeError while deleting execution state",
                       StateException.EX_TYPE_NO_NODE_ERROR), None, sys.exc_info()[2]
   except NotEmptyError as e:
     raise StateException("NotEmptyError while deleting execution state",
                       StateException.EX_TYPE_NOT_EMPTY_ERROR), None, sys.exc_info()[2]
   except ZookeeperError as e:
     raise StateException("Zookeeper while deleting execution state",
                       StateException.EX_TYPE_ZOOKEEPER_ERROR), None, sys.exc_info()[2]
   except Exception as e:
     # Just re raise the exception.
     raise

def get_all_file_state_managers(conf):
  """
  Returns all the file state_managers.
  """
  state_managers = []
  state_locations = conf.get_state_locations_of_type("file")
  for location in state_locations:
    name = location['name']
    rootpath = os.path.expanduser(location['rootpath'])
    LOG.info("Connecting to file state with rootpath: " + rootpath)
    state_manager = FileStateManager(name, rootpath)
    try:
      state_manager.start()
    except Exception as e:
      LOG.error("Exception while connecting to state_manager.")
      traceback.print_exc()
    state_managers.append(state_manager)

  return state_managers

 def delete_pplan(self, topologyName):
   """ delete physical plan info """
   path = self.get_pplan_path(topologyName)
   LOG.info("Removing topology: {0} from path: {1}".format(
       topologyName, path))
   try:
     self.client.delete(path)
     return True
   except NoNodeError:
     raise StateException("NoNodeError while deleting pplan",
                          StateException.EX_TYPE_NO_NODE_ERROR), None, sys.exc_info()[2]
   except NotEmptyError:
     raise StateException("NotEmptyError while deleting pplan",
                          StateException.EX_TYPE_NOT_EMPTY_ERROR), None, sys.exc_info()[2]
   except ZookeeperError:
     raise StateException("Zookeeper while deleting pplan",
                          StateException.EX_TYPE_ZOOKEEPER_ERROR), None, sys.exc_info()[2]
   except Exception:
     # Just re raise the exception.
     raise