Python WMStatsWriter.insertTotalStats方法代码示例

# 需要导入模块: from WMCore.Services.WMStats.WMStatsWriter import WMStatsWriter [as 别名]
# 或者: from WMCore.Services.WMStats.WMStatsWriter.WMStatsWriter import insertTotalStats [as 别名]
class WMStatsTest(unittest.TestCase):
    """
    """
    def setUp(self):
        """
        _setUp_
        """
        self.schema = []
        self.couchApps = ["WMStats"]
        self.testInit = TestInitCouchApp('WorkQueueServiceTest')
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()
        self.testInit.setSchema(customModules = self.schema,
                                useDefault = False)
        self.testInit.setupCouch('wmstats_t', *self.couchApps)
        self.wmstatsWriter = WMStatsWriter(self.testInit.couchUrl, 'wmstats_t');
        return

    def tearDown(self):
        """
        _tearDown_

        Drop all the WMBS tables.
        """
        self.testInit.tearDownCouch()

    def testWMStatsWriter(self):
        # test getWork
        schema = generate_reqmgr_schema()
        self.assertEquals(self.wmstatsWriter.insertRequest(schema[0]), 'OK', 'insert fail');
        self.assertEquals(self.wmstatsWriter.updateRequestStatus(schema[0]['RequestName'], "failed"), 'OK', 'update fail')
        self.assertEquals(self.wmstatsWriter.updateRequestStatus("not_exist_schema", "assigned"),
                          'ERROR: request not found - not_exist_schema')
        self.assertEquals(self.wmstatsWriter.updateTeam(schema[0]['RequestName'], 'teamA'), 'OK', 'update fail')
        self.assertEquals(self.wmstatsWriter.updateTeam("not_exist_schema", 'teamA'),
                          'ERROR: request not found - not_exist_schema')
        totalStats = {'total_jobs': 100, 'input_events': 1000, 'input_lumis': 1234, 'input_num_files': 5}
        self.assertEquals(self.wmstatsWriter.insertTotalStats(schema[0]['RequestName'], totalStats), 'INSERTED', 'update fail')
        self.assertEquals(self.wmstatsWriter.insertTotalStats(schema[0]['RequestName'], totalStats), 'UPDATED', 'update fail')
        self.assertEquals(self.wmstatsWriter.insertTotalStats("not_exist_schema", totalStats),
                          'ERROR: request not found - not_exist_schema')
        spec1 = newWorkload(schema[0]['RequestName'])
        production = spec1.newTask("Production")
        production.setTaskType("Merge")
        production.setSiteWhitelist(['TEST_SITE'])
        self.assertEquals(self.wmstatsWriter.updateFromWMSpec(spec1), 'OK', 'update fail')
        spec2 = newWorkload("not_exist_schema")
        production = spec2.newTask("Production")
        production.setTaskType("Merge")
        self.assertEquals(self.wmstatsWriter.updateFromWMSpec(spec2),
                          'ERROR: request not found - not_exist_schema')

# 需要导入模块: from WMCore.Services.WMStats.WMStatsWriter import WMStatsWriter [as 别名]
# 或者: from WMCore.Services.WMStats.WMStatsWriter.WMStatsWriter import insertTotalStats [as 别名]
    def processInboundWork(self, inbound_work = None, throw = False):
        """Retrieve work from inbox, split and store
        If request passed then only process that request
        """
        if self.params['LocalQueueFlag']:
            self.backend.fixConflicts() # db should be consistent

        result = []
        if not inbound_work:
            inbound_work = self.backend.getElementsForSplitting()
        for inbound in inbound_work:
            # Check we haven't already split the work
            work = self.backend.getElementsForParent(inbound)
            try:
                if work:
                    self.logger.info('Request "%s" already split - Resuming' % inbound['RequestName'])
                else:
                    work, totalStats = self._splitWork(inbound['WMSpec'], None, inbound['Inputs'], inbound['Mask'])
                    self.backend.insertElements(work, parent = inbound) # if this fails, rerunning will pick up here
                    # save inbound work to signal we have completed queueing

                    # add the total work on wmstat summary
                    self.backend.updateInboxElements(inbound.id, Status = 'Acquired')

                    if not self.params.get('LocalQueueFlag') and self.params.get('WMStatsCouchUrl'):
                        # only update global stats for global queue
                        try:
                            wmstatSvc = WMStatsWriter(self.params.get('WMStatsCouchUrl'))
                            wmstatSvc.insertTotalStats(inbound['WMSpec'].name(), totalStats)
                        except Exception, ex:
                            self.logger.info('Error publishing %s to WMStats: %s' % (inbound['RequestName'], str(ex)))

            except TERMINAL_EXCEPTIONS, ex:
                self.logger.info('Failing workflow "%s": %s' % (inbound['RequestName'], str(ex)))
                self.backend.updateInboxElements(inbound.id, Status = 'Failed')
                if throw:
                    raise
            except Exception, ex:
                # if request has been failing for too long permanently fail it.
                # last update time was when element was assigned to this queue
                if (float(inbound.updatetime) + self.params['QueueRetryTime']) < time.time():
                    self.logger.info('Failing workflow "%s" as not queued in %d secs: %s' % (inbound['RequestName'],
                                                                                             self.params['QueueRetryTime'],
                                                                                             str(ex)))
                    self.backend.updateInboxElements(inbound.id, Status = 'Failed')
                else:
                    self.logger.info('Exception splitting work for wmspec "%s": %s' % (inbound['RequestName'], str(ex)))
                if throw:
                    raise
                continue