Python WMAgentDBData.getAgentMonitoring方法代码示例

# 需要导入模块: from WMComponent.AnalyticsDataCollector.DataCollectAPI import WMAgentDBData [as 别名]
# 或者: from WMComponent.AnalyticsDataCollector.DataCollectAPI.WMAgentDBData import getAgentMonitoring [as 别名]

#.........这里部分代码省略.........
        if isDrainMode(self.config):
            logging.info("Agent is in DrainMode")
            agentInfo['drain_mode'] = True
        else:
            agentInfo['drain_mode'] = False

        couchInfo = self.collectCouchDBInfo()
        if couchInfo['status'] != 'ok':
            agentInfo['down_components'].append(couchInfo['name'])
            agentInfo['status'] = couchInfo['status']
            agentInfo['down_component_detail'].append(couchInfo)

        # Disk space warning
        diskUseList = diskUse()
        diskUseThreshold = float(
            self.config.AnalyticsDataCollector.diskUseThreshold)
        agentInfo['disk_warning'] = []
        for disk in diskUseList:
            if float(disk['percent'].strip('%')) >= diskUseThreshold and \
                            disk['mounted'] not in self.config.AnalyticsDataCollector.ignoreDisk:
                agentInfo['disk_warning'].append(disk)

        # Couch process warning
        couchProc = numberCouchProcess()
        logging.info("CouchDB is running with %d processes", couchProc)
        couchProcessThreshold = self.config.AnalyticsDataCollector.couchProcessThreshold
        if couchProc >= couchProcessThreshold:
            agentInfo['couch_process_warning'] = couchProc
        else:
            agentInfo['couch_process_warning'] = 0

        # This adds the last time and message when data was updated to agentInfo
        lastDataUpload = DataUploadTime.getInfo()
        if lastDataUpload['data_last_update']:
            agentInfo['data_last_update'] = lastDataUpload['data_last_update']
        if lastDataUpload['data_error']:
            agentInfo['data_error'] = lastDataUpload['data_error']

        # Change status if there is data_error, couch process maxed out or disk full problems.
        if agentInfo['status'] == 'ok' and (agentInfo['drain_mode']
                                            or agentInfo['disk_warning']):
            agentInfo['status'] = "warning"

        if agentInfo['status'] == 'ok' or agentInfo['status'] == 'warning':
            if agentInfo.get('data_error', 'ok') != 'ok' or agentInfo.get(
                    'couch_process_warning', 0):
                agentInfo['status'] = "error"

        if agentInfo['down_components']:
            logging.info("List of agent components down: %s" %
                         agentInfo['down_components'])

        return agentInfo

    def uploadAgentInfoToCentralWMStats(self, agentInfo, uploadTime):
        #direct data upload to the remote to prevent data conflict when agent is cleaned up and redeployed
        agentDocs = convertToAgentCouchDoc(agentInfo, self.config.ACDC,
                                           uploadTime)
        self.centralWMStatsCouchDB.updateAgentInfo(agentDocs)

    def collectWMBSInfo(self):
        """
        Fetches WMBS job information.
        In addition to WMBS, also collects RunJob info from BossAir
        :return: dict with the number of jobs in each status
        """
        results = {}
        logging.info("Getting wmbs job info ...")
        # first retrieve the site thresholds
        results['thresholds'] = self.wmagentDB.getJobSlotInfo()
        logging.info("Running and pending site thresholds: %s",
                     results['thresholds'])

        # now fetch the amount of jobs in each state and the amount of created
        # jobs grouped by task
        results.update(self.wmagentDB.getAgentMonitoring())
        logging.info("Total number of jobs in WMBS sorted by status: %s",
                     results['wmbsCountByState'])
        logging.info(
            "Total number of 'created' jobs in WMBS sorted by type: %s",
            results['wmbsCreatedTypeCount'])
        logging.info(
            "Total number of 'executing' jobs in WMBS sorted by type: %s",
            results['wmbsExecutingTypeCount'])

        logging.info(
            "Total number of active jobs in BossAir sorted by status: %s",
            results['activeRunJobByStatus'])
        logging.info(
            "Total number of complete jobs in BossAir sorted by status: %s",
            results['completeRunJobByStatus'])

        logging.info(
            "Available slots thresholds to pull work from GQ to LQ: %s",
            results['thresholdsGQ2LQ'])
        logging.info(
            "List of jobs pending for each site, sorted by priority: %s",
            results['sitePendCountByPrio'])

        return results

# 需要导入模块: from WMComponent.AnalyticsDataCollector.DataCollectAPI import WMAgentDBData [as 别名]
# 或者: from WMComponent.AnalyticsDataCollector.DataCollectAPI.WMAgentDBData import getAgentMonitoring [as 别名]

#.........这里部分代码省略.........
        else:
            agentInfo['drain_mode'] = False

        couchInfo = self.collectCouchDBInfo()
        if couchInfo['status'] != 'ok':
            agentInfo['down_components'].append(couchInfo['name'])
            agentInfo['status'] = couchInfo['status']
            agentInfo['down_component_detail'].append(couchInfo)

        # Couch process warning
        couchProc = numberCouchProcess()
        logging.info("CouchDB is running with %d processes", couchProc)
        couchProcessThreshold = self.config.AnalyticsDataCollector.couchProcessThreshold
        if couchProc >= couchProcessThreshold:
            agentInfo['couch_process_warning'] = couchProc
        else:
            agentInfo['couch_process_warning'] = 0

        # This adds the last time and message when data was updated to agentInfo
        lastDataUpload = DataUploadTime.getInfo()
        if lastDataUpload['data_last_update']:
            agentInfo['data_last_update'] = lastDataUpload['data_last_update']
        if lastDataUpload['data_error']:
            agentInfo['data_error'] = lastDataUpload['data_error']

        # Change status if there is data_error, couch process maxed out or disk full problems.
        if agentInfo['status'] == 'ok' and (agentInfo['drain_mode'] or agentInfo['disk_warning']):
            agentInfo['status'] = "warning"

        if agentInfo['status'] == 'ok' or agentInfo['status'] == 'warning':
            if agentInfo.get('data_error', 'ok') != 'ok' or agentInfo.get('couch_process_warning', 0):
                agentInfo['status'] = "error"

        logging.info("List of agent components down: %s", agentInfo['down_components'])

        return agentInfo

    def uploadAgentInfoToCentralWMStats(self, agentInfo, uploadTime):
        # direct data upload to the remote to prevent data conflict when agent is cleaned up and redeployed
        agentDocs = convertToAgentCouchDoc(agentInfo, self.config.ACDC, uploadTime)
        self.centralWMStatsCouchDB.updateAgentInfo(agentDocs)

    @timeFunction
    def collectWMBSInfo(self):
        """
        Fetches WMBS job information.
        In addition to WMBS, also collects RunJob info from BossAir
        :return: dict with the number of jobs in each status
        """
        logging.info("Getting wmbs job info ...")
        results = {}

        # first retrieve the site thresholds
        results['thresholds'] = self.wmagentDB.getJobSlotInfo()
        logging.debug("Running and pending site thresholds: %s", results['thresholds'])

        # now fetch the amount of jobs in each state and the amount of created
        # jobs grouped by task
        results.update(self.wmagentDB.getAgentMonitoring())

        logging.debug("Total number of jobs in WMBS sorted by status: %s", results['wmbsCountByState'])
        logging.debug("Total number of 'created' jobs in WMBS sorted by type: %s", results['wmbsCreatedTypeCount'])
        logging.debug("Total number of 'executing' jobs in WMBS sorted by type: %s", results['wmbsExecutingTypeCount'])

        logging.debug("Total number of active jobs in BossAir sorted by status: %s", results['activeRunJobByStatus'])
        logging.debug("Total number of complete jobs in BossAir sorted by status: %s",
                      results['completeRunJobByStatus'])

        logging.debug("Available slots thresholds to pull work from GQ to LQ: %s", results['thresholdsGQ2LQ'])
        logging.debug("List of jobs pending for each site, sorted by priority: %s", results['sitePendCountByPrio'])

        return results

    def checkProxyLifetime(self, agInfo):
        """
        Check the proxy lifetime (usually X509_USER_CERT) and raise either
        a warning or an error if the proxy validity is about to expire.
        :param agInfo: dictionary with plenty of agent monitoring information in place.
        :return: same dictionary object plus additional keys/values if needed.
        """
        secsLeft = self.proxy.getTimeLeft(proxy=self.proxyFile)
        logging.debug("Proxy '%s' lifetime is %d secs", self.proxyFile, secsLeft)


        if secsLeft <= 86400 * 3:  # 3 days
            proxyWarning = True
            agInfo['status'] = "error"
        elif secsLeft <= 86400 * 5:  # 5 days
            proxyWarning = True
            if agInfo['status'] == "ok":
                agInfo['status'] = "warning"
        else:
            proxyWarning = False

        if proxyWarning:
            warnMsg = "Agent proxy '%s' must be renewed ASAP. " % self.proxyFile
            warnMsg += "Its time left is: %.2f hours." % (secsLeft / 3600.)
            agInfo['proxy_warning'] = warnMsg

        return

# 需要导入模块: from WMComponent.AnalyticsDataCollector.DataCollectAPI import WMAgentDBData [as 别名]
# 或者: from WMComponent.AnalyticsDataCollector.DataCollectAPI.WMAgentDBData import getAgentMonitoring [as 别名]

#.........这里部分代码省略.........

    def uploadAgentInfoToCentralWMStats(self, agentInfo):
        """
        Add some required fields to the document before it can get uploaded
        to WMStats.
        :param agentInfo: dict with agent stats to be posted to couchdb
        """
        agentInfo['_id'] = agentInfo["agent_url"]
        agentInfo['timestamp'] = int(time.time())
        agentInfo['type'] = "agent_info"
        # directly upload to the remote to prevent data conflict when agent is cleaned up and redeployed
        try:
            self.centralWMStatsCouchDB.updateAgentInfo(agentInfo,
                                                       propertiesToKeep=["data_last_update", "data_error"])
        except Exception as e:
            logging.error("Failed to upload agent statistics to WMStats. Error: %s", str(e))

    @timeFunction
    def collectWMBSInfo(self):
        """
        Fetches WMBS job information.
        In addition to WMBS, also collects RunJob info from BossAir
        :return: dict with the number of jobs in each status
        """
        logging.info("Getting wmbs job info ...")
        results = {}

        # first retrieve the site thresholds
        results['thresholds'] = self.wmagentDB.getJobSlotInfo()
        logging.debug("Running and pending site thresholds: %s", results['thresholds'])

        # now fetch the amount of jobs in each state and the amount of created
        # jobs grouped by task
        results.update(self.wmagentDB.getAgentMonitoring())

        logging.debug("Total number of jobs in WMBS sorted by status: %s", results['wmbsCountByState'])
        logging.debug("Total number of 'created' jobs in WMBS sorted by type: %s", results['wmbsCreatedTypeCount'])
        logging.debug("Total number of 'executing' jobs in WMBS sorted by type: %s", results['wmbsExecutingTypeCount'])

        logging.debug("Total number of active jobs in BossAir sorted by status: %s", results['activeRunJobByStatus'])
        logging.debug("Total number of complete jobs in BossAir sorted by status: %s",
                      results['completeRunJobByStatus'])

        logging.debug("Available slots thresholds to pull work from GQ to LQ: %s", results['thresholdsGQ2LQ'])
        logging.debug("List of jobs pending for each site, sorted by priority: %s", results['sitePendCountByPrio'])

        return results

    def checkCredLifetime(self, agInfo, credType):
        """
        Check the credential lifetime. Usually X509_USER_PROXY or X509_USER_CERT
        and raise either a warning or an error if the proxy validity is about to expire.
        :param agInfo: dictionary with plenty of agent monitoring information in place.
        :param credType: credential type, can be: "proxy" or "certificate"
        :return: same dictionary object plus additional keys/values if needed.
        """
        if credType == "proxy":
            credFile = self.proxyFile
            secsLeft = self.proxy.getTimeLeft(proxy=credFile)
        elif credType == "certificate":
            credFile = self.userCertFile
            secsLeft = self.proxy.getUserCertTimeLeft(openSSL=True)
        else:
            logging.error("Unknown credential type. Available options are: [proxy, certificate]")
            return