Python util.E类代码示例

def SyncOneboxLog(config):
  """Syncs Local Onebox log file with GFS Onebox Log file ONLY on clusters.
  As of 4.6.4, this is called from scripts/periodic_script.py and from
  onebox_handler.py, when the user does View Log AND the machine is a cluster.
  """
  onebox_port = servertype.GetPortBase('oneboxenterprise')
  onebox_node = config.SERVERS[onebox_port]
  crt_machine = E.getCrtHostName()
  ent_config_type = config.var('ENT_CONFIG_TYPE')

  #If onebox server is not running no need to sync.
  if ent_config_type != 'CLUSTER' or crt_machine != onebox_node[0]:
    return

  tmp_dir     = config.var('TMPDIR')
  gfs_cell    = config.var('GFS_CELL')
  local_log_name = os.path.join(tmp_dir, config.var('ENTERPRISE_ONEBOX_LOG'))
  gfs_log_name   = os.path.join(os.sep, 'gfs', gfs_cell,
                                config.var('ENTERPRISE_ONEBOX_LOG'))

  equalize_command = 'equalize %s %s' % (local_log_name, gfs_log_name)

  # fileutil equalize copies only the difference of the log files.
  err, out = E.run_fileutil_command(config, equalize_command)
  if not err:
    return

  # files didn't match in the begining, possibly a new log file would have
  # created, copy the whole log file in such case.
  copy_command = 'cp -f %s %s' % (local_log_name, gfs_log_name)
  err, out = E.run_fileutil_command(config, copy_command)

  if err:
    logging.error('Error while syncing onebox logs.')

  def browse(self, clientName, virtualFile, fromLine, toLine,
             grepString, fileArgs):

    grepString = urllib.quote_plus(grepString.strip())
    fromLine = string.atoi(fromLine)
    toLine   = string.atoi(toLine)

    fileLocation = self.makePhysicalFile(virtualFile, clientName, grepString,
                                         fileArgs)
    if not fileLocation:
      raise ar_exception.ARException((
        ar_exception.LOADPARAMS, M.ERR_INVALIDFILESPECIFICATION))

    # Get valid start line and end line
    numLines = toLine - fromLine + 1;
    if numLines < 0:  numLines = 0
    toLine = fromLine + numLines - 1

    # Get the lines
    result = []

    parsedGrepString = commands.mkarg(grepString)
    if ( E.ERR_OK != E.execute(
        [E.getCrtHostName()], "head -n %s %s | tail -n %s | grep -i -F -- %s" %
        (toLine, fileLocation, numLines, parsedGrepString), result, false) ):
      return "0"

    return "%s\n%s" % (len(result[0]), result[0])

def SyncOpLogs(all_machines, log_dir):
  """ This will sync the AdminRunner.OPERATOR.* logs to all machines """

  # We have to run this only on master
  master = find_master.FindMaster(2100, all_machines)

  # The name of this machine
  crt_machine = E.getCrtHostName()
  if len(master) == 1 and master[0] == crt_machine:
    for machine in all_machines:
      if machine != crt_machine:
        src_dir = '%s/AdminRunner.OPERATOR.*' % (log_dir)
        dest_dir = '%s:/%s' % (machine, log_dir)

        logging.info('Collecting operator logs from %s into %s' % (
          src_dir, dest_dir))

        rsync_cmd = 'rsync --timeout=20 --size-only -vau ' \
                     ' -e ssh %s %s/' % (src_dir, dest_dir)

        # rsync the logs
        lockfile = '%s/syncops_lock' % log_dir
        lock = E.acquire_lock(lockfile, 1, breakLockAfterGracePeriod = 0)
        if lock == None:
          logging.info('Cannot grab the lock. Return!')
          return

        try:
          (status, output) = liblog.DoCommand(rsync_cmd)
          if status != 0:
            logging.error('Failed to collect logs from %s: %s' % (
              machine, output))
        finally:
          lock.close()
          os.unlink(lockfile)

def SyncLogsWithCanonical(ver, canonical):
  """ sync logs with the canonical

  Arguments:
    ver: '4.6.5'
    canonical: 'ent1'
  """

  gfs_master_nodes, _ = core_utils.GFSMasterNodes()
  gfs_master_nodes.remove(canonical)
  gfs_master_dir = '/export/hda3/%s/data/gfs_master' % ver
  log_dir = '%s/ent.gfsmaster' % gfs_master_dir
  backup_log_dir = '%s.backup.%d' % (log_dir, int(time.time()))
  vars = {'gfs_master_dir':     gfs_master_dir,
          'log_dir':            log_dir,
          'backup_log_dir':     backup_log_dir,
          'canonical':          canonical
         }
  cmd = ('rm -rf %(log_dir)s.backup*; '
         'mv %(log_dir)s %(backup_log_dir)s; '
         'mkdir -p %(log_dir)s; chown nobody:nobody %(log_dir)s; '
         'rsync -c -e ssh -aHpogt %(canonical)s:%(log_dir)s %(gfs_master_dir)s'
         % vars
        )
  out = []
  enthome = '/export/hda3/%s' % ver
  E.execute(gfs_master_nodes, cmd, out, 1200, 1, 0, enthome)

def SetInitState(cfg, state):
  """Sets system's initialization state. For oneway, it stores it in
  C.ENT_SYSTEM_INIT_STATE. For Clusters, it stores it in chubby file
  /ls/ent<version>/ENT_SYSTEM_INIT_STATE.

  @param cfg - of type configurator.
  @param state - string
  """
  # oneway?
  if 1 == len(core_utils.GetNodes()):
    cfg.setGlobalParam(C.ENT_SYSTEM_INIT_STATE, state)
    return

  tmpfile = E.mktemp('/export/hda3/tmp')
  try:
    f = open(tmpfile, 'w')
    f.write(state)
    f.close()
  except IOError:
    logging.fatal('Cannot write to temp file %s' % tmpfile)
    return
  version = cfg.getGlobalParam('VERSION')
  lockserv_cmd_prefix = core_utils.GetLSClientCmd(version, is_test(version))
  chubby_root_dir = '/ls/%s' % core_utils.GetCellName(version)
  write_cmd =  '%s cp %s %s/%s' % (lockserv_cmd_prefix,
      tmpfile, chubby_root_dir, 'ENT_SYSTEM_INIT_STATE')
  logging.info('setting system init state to: %s', state)
  E.exe_or_fail(write_cmd)
  E.exe('rm -rf %s' % tmpfile)

def kill_service(services, machines):
  """Kill all processes associated with specified services on the specified
  machines. E.execute() sends the commands concurrently when there is more than
  one node.

  Args:
    services: list of services to kill. 'adminconsole' and 'adminrunner' are
              currently supported.
    machines: list of hostnames
  """
  # Map of services to the command that kills the service
  find_service_pid_cmd = {
      'adminconsole': ("ps -e -o pid,args --width 100 | "
                       "grep loop_AdminConsole.py | grep -v grep | "
                       "awk '{print $1}' ; "
                       "%s" % python_kill.GetServicesListeningOn(['8000'])),
      'adminrunner': ("ps -e -o pid,args --width 100 | "
                      "grep loop_AdminRunner.py | grep -v grep | "
                      "awk '{print $1}' ; "
                      "%s" % python_kill.GetServicesListeningOn(['2100'])),
  }

  for service in services:
    if service not in find_service_pid_cmd:
      logging.error('kill_service: Unrecognised service "%s"' % service)
    else:
      logging.info('kill_service: Killing service "%s" on %d nodes...' %
                   (service, len(machines)))
      kill_cmd = ('sh -c "(kill `%s`; sleep 3; kill -9 `%s`; true)" '
                  '> /dev/null 2>&1' %
                  (find_service_pid_cmd[service],
                   find_service_pid_cmd[service]))
      E.execute(machines, kill_cmd, [], alarm=1, verbose=0)
      logging.info('kill_service: Done killing service "%s"' % service)

def CollectLogs(all_machines, gws_log_dir, log_collect_dir):
  # We only run this on oneway or master node of cluster.
  master = find_master.FindMaster(2100, all_machines)
  crt_machine = E.getCrtHostName()
  if len(all_machines) != 1 and (len(master) != 1 or master[0] != crt_machine):
    logging.info('Not a oneway or cluster master node. Return!')
    return

  lockfile = '%s/lock' % log_collect_dir
  # waiting up to 5 minutes for the lock.
  lock = E.acquire_lock(lockfile, 30, breakLockAfterGracePeriod = 0)
  if lock == None:
    logging.info('Cannot grab the lock. Return!')
    return

  try:
    for machine in all_machines:
      src_pattern = '%s/partnerlog.*' % gws_log_dir
      dest_dir = '%s/%s' % (log_collect_dir, machine)

      # If it's a oneway or master node, we make a symlink to gws_log_dir instead
      # of rsync to log_collect directory
      if machine == crt_machine:
        # To make it backward compatible, we need to remove old dest_dir if it's
        # already an existing directory from previous version because in previous
        # versions we created a dir and rsynced files even on the master node and
        # one-ways.
        if os.path.exists(dest_dir) and not os.path.islink(dest_dir):
          if not E.rm(master, '%s/*' % dest_dir) or not E.rmdir(master, dest_dir):
            logging.error('Directory %s exists and cannot be cleaned.', dest_dir)
            continue
          logging.info('Cleaned existing directory %s.', dest_dir)

        if E.ln(master, gws_log_dir, dest_dir):
          logging.info('Symlink %s to directory %s:%s for logs' %
                       (dest_dir, machine, gws_log_dir))
        else:
          logging.error('Cannot make a symlink from %s to %s' %
                        (dest_dir, gws_log_dir))
        continue

      # For non-master nodes on cluster, we need to rsync those files to master node
      logging.info('Collecting logs from %s:%s into %s' % (
        machine, src_pattern, dest_dir))

      # make log directories if needed
      liblog.MakeDir(dest_dir)

      # rsync all files from one remote machine in one command.
      rsync_cmd = 'rsync --timeout=60 --size-only -vau ' \
                  ' -e ssh %s:%s %s/' % (machine, src_pattern, dest_dir)

      # rsync the logs
      (status, output) = liblog.DoCommand(rsync_cmd)
      if status != 0:
        logging.error('Failed to collect logs from %s: %s' % (
          machine, output))
  finally:
    lock.close()
    os.unlink(lockfile)

def main(unused_args):
  if not E.access([E.LOCALHOST], FLAGS.enthome, 'rd'):
    sys.exit("Invalid enthome %s" % FLAGS.enthome)

  if ( not FLAGS.box_keys_dir or
       not E.access([E.LOCALHOST], FLAGS.box_keys_dir, 'rwd') ):
    sys.exit("Invalid box_keys_dir %s" % FLAGS.box_keys_dir)
  if ( not FLAGS.license_keys_dir or
       not E.access([E.LOCALHOST], FLAGS.license_keys_dir, 'rwd') ):
    sys.exit("Invalid license_keys_dir %s" % FLAGS.license_keys_dir)

  if FLAGS.installstate not in ["INSTALL", "ACTIVE", "SERVE", "TEST"]:
    sys.exit("Invalid --installstate %s" % FLAGS.installstate)

  reset_index.SetResetStatusCacheTimeout(FLAGS.reset_status_cache_timeout)

  as = adminrunner_server.AdminRunnerServer(FLAGS.enthome,
                                            FLAGS.installstate,
                                            FLAGS.port,
                                            FLAGS.box_keys_dir,
                                            FLAGS.license_keys_dir)

  # make sure we have been given a config file
  if (not as.cfg.getGlobalParam(C.ENT_SYSTEM_HAS_VALID_CONFIG) and
      as.cfg.getInstallState() != 'INSTALL'):
    logging.fatal("adminrunner doesn't have a config file; you must "\
                  "install a conf rpm or run it with "\
                  "--installstate=INSTALL (for migration)")
    return # just in case fatal doesn't exit

  def deleteCollection(self, collection):
    """Delete all reports and logs for a particular collection."""
    self.logreplock.acquire()
    try:
      for reportType in [liblog.RAW_REPORT, liblog.SUMMARY_REPORT]:
        reports = self.getLogReports(collection, reportType)
        for report in reports:
          # stop running job if report is being (re)generated.
          if report.completeState != COMPLETE:
            self.stopRunningJob(self.jobName(report))

          # delete data files if any.
          (html_file, valid_file) = liblog.get_report_filenames(self.entConfig,
                                     reportType, report.reportName, collection)
          self.RemoveReportFiles(html_file, valid_file)
        self.reportCount[reportType] -= len(reports)
        logging.info('Delete total %d reports of type %s for collection %s.' % (
          len(reports), reportType, collection))
        listfile = liblog.get_report_list_filename(self.entConfig,
                                                   reportType, collection)
        (err, out) = E.run_fileutil_command(self.entConfig,
                                            'rm -f %s' % listfile)
        if err:
          logging.error('Cannot remove list file %s.' % listfile)

      report_collection_dir = liblog.get_report_collection_dir(self.entConfig,
                                                               collection)
      (err, out) = E.run_fileutil_command(self.entConfig,
                                          'rmdir %s' % report_collection_dir)
      if err:
        logging.error('Cannot delete unused directory %s' % \
                      report_collection_dir)
    finally:
      self.logreplock.release()

    def setcert(self, certBody):
        """ Takes a cert file body as the input, and saves it
    as the staging certificate
    returns 0 on success, or 1 on failure
    """

        retval = 0
        self.updatelock.acquire()
        try:
            try:
                open(ssl_cert.STAGINGCERT_FILENAME % self.cfg.getGlobalParam("ENTERPRISE_HOME"), "w").write(certBody)
            except IOError:
                retval = 1
                logging.error(
                    "Couldn't save certificate to [%s]"
                    % (ssl_cert.STAGINGCERT_FILENAME % self.cfg.getGlobalParam("ENTERPRISE_HOME"))
                )

            if retval == 0:
                verifycmd = "secure_script_wrapper -p2 %s verifystagingcert %s" % (
                    self.sslWrapperPath,
                    self.cfg.getGlobalParam("ENTERPRISE_HOME"),
                )
                outputList = []
                verifycode = E.execute(["localhost"], verifycmd, outputList, 60)

                if verifycode != 0:
                    retval = 1
                    E.rm(["localhost"], ssl_cert.STAGINGCERT_FILENAME % self.cfg.getGlobalParam("ENTERPRISE_HOME"))
                    logging.error("Couldn't verify certificate [%s]; error code: %d" % (str(outputList), verifycode))

        finally:
            self.updatelock.release()

        return "%d" % retval

def KillProcessIfNotMaster(config):
  # kill babysitter
  util_dir = "%s/local/google3/enterprise/legacy/util" % config.ENTERPRISE_HOME
  E.killBabysitter(util_dir, config.GetConfigFileName(), config.VERSION)

  # stop snmpd when not a master
  disableSnmp()

 def replicateConfig(self, machines):
   """ Replicates the config to a list of machines """
   cmd = (". %s && cd %s/local/google3/enterprise/legacy/scripts && "
          "./replicate_config.py %s %s" % (
     self.getGlobalParam("ENTERPRISE_BASHRC"), self.entHome,
     E.getCrtHostName(), self.globalParams.GetConfigFileName()))
   return E.execute(machines, cmd, None, true)

def _RunServeCmd(cfg, version, cmd, allnodes=0):
  """Run serve_service command.
  cmd: 'stop', 'start', 'activate', 'deactivate'
  allnodes: 1 to run command on all nodes
  """
  serve_service_cmd = (
      '/export/hda3/%s/local/google3/enterprise/legacy/scripts/'
      'serve_service.py %s %s' % (version,
                                  cfg.getGlobalParam('ENTERPRISE_HOME'),
                                  cmd))
  logging.info('Running: %s' % serve_service_cmd)
  if allnodes:
    machines = cfg.getGlobalParam(C.MACHINES)
  else:
    machines = [E.getCrtHostName()]

  if E.execute(machines,
                   SECURE_WRAPPER_COMMAND % ( \
                        cfg.getGlobalParam('ENTERPRISE_HOME'),
                        '-p2',
                        serve_service_cmd),
                   None, 0) != E.ERR_OK:
    logging.error('%s: failed' % serve_service_cmd)
    return 1
  logging.info('%s: completed' % serve_service_cmd)
  return 0

  def drain_urlmanagers(self):
    """
    We need to do this before advancing the epoch -- we can do it
    multiple times
    """
    urlmanagers = self.cfg.globalParams.GetServerHostPorts("urlmanager")
    num_shards = self.cfg.globalParams.GetNumShards('urlmanager')
    epoch = self.cfg.getGlobalParam('RT_EPOCH')

    for (host, port) in urlmanagers:
      # We don't do it here directly because of the timeout
      cmd = ". %s; cd %s/local/google3/enterprise/legacy/util && "\
            "./port_talker.py %s %d 'd DumpingStatusTable' %d" % (
          self.cfg.getGlobalParam('ENTERPRISE_BASHRC'),
          self.cfg.entHome,
          host, port, 300) # 5 min timeout
      err = E.execute([E.getCrtHostName()], cmd, None, 0)
      if E.ERR_OK != err:
        logging.error("Error draining urlmanagers [%s]" % err)
        return 1

      # Make sure that the file is out
      shard_num = servertype.GetPortShard(port)
      file = "%surlmanager_out_table_%02d_of_%02d_epoch%010d" % (
        self.cfg.getGlobalParam('NAMESPACE_PREFIX'),
        shard_num, num_shards, epoch)
      err, out = E.run_fileutil_command(self.cfg.globalParams, "ls %s" % file)
      if E.ERR_OK != err:
        logging.error("The status table file [%s] is not there" % file)
        return 1

    return 0

    def installkey(self):
        """ installs the staging key as the currently installed private key
    returns:
    0 on success,
    1 on empty install key (not an error)
    2 when the private key is invalid
    3 when the private key could not be distributed
    """

        self.updatelock.acquire()
        try:

            # first verify if the staging key is empty (not an error)
            if (
                not os.path.exists(ssl_cert.STAGINGKEY_FILENAME % self.cfg.getGlobalParam("ENTERPRISE_HOME"))
            ) or 0 == len(open(ssl_cert.STAGINGKEY_FILENAME % self.cfg.getGlobalParam("ENTERPRISE_HOME"), "r").read()):
                return "1"

            # next verify that the staging key is a valid file
            verifycmd = "secure_script_wrapper -p2 %s verifystagingkey %s" % (
                self.sslWrapperPath,
                self.cfg.getGlobalParam("ENTERPRISE_HOME"),
            )
            outputList = []
            verifycode = E.execute(["localhost"], verifycmd, outputList, 60)

            if verifycode != 0:
                E.rm(["localhost"], ssl_cert.STAGINGKEY_FILENAME % self.cfg.getGlobalParam("ENTERPRISE_HOME"))
                logging.error("Verify failed for key [%s]; error code: %d" % (str(outputList), verifycode))
                return "2"

            # distribute the staging key
            retcode = E.distribute(
                self.cfg.getGlobalParam("MACHINES"),
                ssl_cert.STAGINGKEY_FILENAME % self.cfg.getGlobalParam("ENTERPRISE_HOME"),
                60,
            )
            if retcode != 0:
                logging.error("Couldn't distribute private key, error %d" % retcode)
                return "3"

            # next, copy the key on all machines
            cmd = "secure_script_wrapper -p2 %s installkey %s" % (
                self.sslWrapperPath,
                self.cfg.getGlobalParam("ENTERPRISE_HOME"),
            )

            outputList = []
            retcode = E.execute(self.cfg.getGlobalParam("MACHINES"), cmd, outputList, 60)

            if retcode != 0:
                logging.error("Couldn't install cert: %s" % str(outputList))
                return "3"

            self.writeAdminRunnerOpMsg(M.MSG_LOG_SSL_KEY_INSTALLED)
        finally:
            self.updatelock.release()

        return "0"