Python WMStatsWriter.updateRequestStatus方法代码示例

# 需要导入模块: from WMCore.Services.WMStats.WMStatsWriter import WMStatsWriter [as 别名]
# 或者: from WMCore.Services.WMStats.WMStatsWriter.WMStatsWriter import updateRequestStatus [as 别名]
class WMStatsTest(unittest.TestCase):
    """
    """
    def setUp(self):
        """
        _setUp_
        """
        self.schema = []
        self.couchApps = ["WMStats"]
        self.testInit = TestInitCouchApp('WorkQueueServiceTest')
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()
        self.testInit.setSchema(customModules = self.schema,
                                useDefault = False)
        self.testInit.setupCouch('wmstats_t', *self.couchApps)
        self.wmstatsWriter = WMStatsWriter(self.testInit.couchUrl, 'wmstats_t');
        return

    def tearDown(self):
        """
        _tearDown_

        Drop all the WMBS tables.
        """
        self.testInit.tearDownCouch()

    def testWMStatsWriter(self):
        # test getWork
        schema = generate_reqmgr_schema()
        self.assertEquals(self.wmstatsWriter.insertRequest(schema[0]), 'OK', 'insert fail');
        self.assertEquals(self.wmstatsWriter.updateRequestStatus(schema[0]['RequestName'], "failed"), 'OK', 'update fail')
        self.assertEquals(self.wmstatsWriter.updateRequestStatus("not_exist_schema", "assigned"),
                          'ERROR: request not found - not_exist_schema')
        self.assertEquals(self.wmstatsWriter.updateTeam(schema[0]['RequestName'], 'teamA'), 'OK', 'update fail')
        self.assertEquals(self.wmstatsWriter.updateTeam("not_exist_schema", 'teamA'),
                          'ERROR: request not found - not_exist_schema')
        totalStats = {'total_jobs': 100, 'input_events': 1000, 'input_lumis': 1234, 'input_num_files': 5}
        self.assertEquals(self.wmstatsWriter.insertTotalStats(schema[0]['RequestName'], totalStats), 'INSERTED', 'update fail')
        self.assertEquals(self.wmstatsWriter.insertTotalStats(schema[0]['RequestName'], totalStats), 'UPDATED', 'update fail')
        self.assertEquals(self.wmstatsWriter.insertTotalStats("not_exist_schema", totalStats),
                          'ERROR: request not found - not_exist_schema')
        spec1 = newWorkload(schema[0]['RequestName'])
        production = spec1.newTask("Production")
        production.setTaskType("Merge")
        production.setSiteWhitelist(['TEST_SITE'])
        self.assertEquals(self.wmstatsWriter.updateFromWMSpec(spec1), 'OK', 'update fail')
        spec2 = newWorkload("not_exist_schema")
        production = spec2.newTask("Production")
        production.setTaskType("Merge")
        self.assertEquals(self.wmstatsWriter.updateFromWMSpec(spec2),
                          'ERROR: request not found - not_exist_schema')

# 需要导入模块: from WMCore.Services.WMStats.WMStatsWriter import WMStatsWriter [as 别名]
# 或者: from WMCore.Services.WMStats.WMStatsWriter.WMStatsWriter import updateRequestStatus [as 别名]
def changeRequestStatus(requestName, newState, priority=None, wmstatUrl=None):
    """
    _changeRequestStatus_

    Basic API to change a request to a new state, also
    includes optional priority change for the request

    - *requestName* : name of the request to be modified
    - *newState*    : name of the new status for the request
    - *priority* : optional integer priority
    
    Apparently when changing request state (on assignment page),
    it's possible to change priority at one go. Hence the argument is here.

    """
    # MySQL/Oracle
    factory = DBConnect.getConnection()
    reqId = getRequestID(factory, requestName)
    changeRequestIDStatus(reqId, newState, priority)

    # CouchDB
    # have to first get information where the request Couch document is,
    # extracting the information from reqmgr_request.workflow table field
    reqData = factory(classname="Request.Get").execute(reqId)
    # this would be something like this:
    # http://localhost:5984/reqmgr_workload_cache/maxa_RequestString-OVERRIDE-ME_130306_205649_8066/spec
    wfUrl = reqData["workflow"]
    # cut off /maxa_RequestString-OVERRIDE-ME_130306_205649_8066/spec
    couchUrl = wfUrl.replace("/" + requestName + "/spec", "")
    couchDbName = couchUrl[couchUrl.rfind("/") + 1 :]
    # cut off database name from the URL
    url = couchUrl.replace("/" + couchDbName, "")
    couchDb = Database(couchDbName, url)
    fields = {"RequestStatus": newState}
    couchDb.updateDocument(requestName, "ReqMgr", "updaterequest", fields=fields, useBody=True)

    # TODO: should we make this mendatory?
    if wmstatUrl:
        wmstatSvc = WMStatsWriter(wmstatUrl)
        wmstatSvc.updateRequestStatus(requestName, newState)

# 需要导入模块: from WMCore.Services.WMStats.WMStatsWriter import WMStatsWriter [as 别名]
# 或者: from WMCore.Services.WMStats.WMStatsWriter.WMStatsWriter import updateRequestStatus [as 别名]
def changeRequestStatus(requestName, newState, priority = None, wmstatUrl = None):
    """
    _changeRequestStatus_

    Basic API to change a request to a new state, also
    includes optional priority change for the request

    - *requestName* : name of the request to be modified
    - *newState*    : name of the new status for the request
    - *priority* : optional integer priority
    
    Apparently when changing request state (on assignment page),
    it's possible to change priority at one go. Hence the argument is here.

    """
    #TODO: should we make this mendatory?
    if wmstatUrl:
        wmstatSvc = WMStatsWriter(wmstatUrl)
        wmstatSvc.updateRequestStatus(requestName, newState)

    factory = DBConnect.getConnection()
    reqId = getRequestID(factory, requestName)
    changeRequestIDStatus(reqId, newState, priority)
    return

# 需要导入模块: from WMCore.Services.WMStats.WMStatsWriter import WMStatsWriter [as 别名]
# 或者: from WMCore.Services.WMStats.WMStatsWriter.WMStatsWriter import updateRequestStatus [as 别名]
        print("AnalyticsDataCollector.centralWMStatsURL is not specified")
        sys.exit(1)
        
    parser = OptionParser()
    parser.set_usage("wmstats-request-status-chagne [agent_url:port]")
    
    parser.add_option("-r", "--request", dest = "request",
                      help = "resquest name")
    
    parser.add_option("-s", "--status", dest = "newstatus",
                      help = "set to new status")
    
    (options, args) = parser.parse_args()
    
    if not options.request:
        print("request name needs to be set")
        sys.exit(1)
    
    if not options.newstatus:
        print("new status needs to be set")
        sys.exit(1)
    
    answer = raw_input("%s change to %s in wmstats db (yes, no)?" % (options.request, options.newstatus))
    if not answer.lower() == "yes":
        print("Canceled")
        sys.exit(1)
    
    report = wmstats.updateRequestStatus(options.request, options.newstatus)
    
    print(report)

# 需要导入模块: from WMCore.Services.WMStats.WMStatsWriter import WMStatsWriter [as 别名]
# 或者: from WMCore.Services.WMStats.WMStatsWriter.WMStatsWriter import updateRequestStatus [as 别名]
class Tier0PluginTest(unittest.TestCase):


    def setUp(self):
        """
        _setUp_

        Setup the test environment
        """
        self.testInit = TestInit(__file__)
        self.testInit.setDatabaseConnection()
        self.testInit.setSchema(["WMCore.WMBS"])
        self.wmstatsCouchDB = 'wmstats_plugin_t'
        self.testInit.setupCouch(self.wmstatsCouchDB, 'WMStats')
        self.testDir = self.testInit.generateWorkDir()

        self.wmstatsWriter = WMStatsWriter(os.environ['COUCHURL'], self.wmstatsCouchDB)

        self.stateMap = {}
        self.orderedStates = []
        self.plugin = None

        return

    def tearDown(self):
        """
        _tearDown_

        Clear databases and delete files
        """
        self.testInit.tearDownCouch()
        self.testInit.clearDatabase()
        self.testInit.delWorkDir()

        return

    def setupRepackWorkflow(self):
        """
        _setupRepackWorkflow_

        Populate WMBS with a repack-like workflow,
        every subscription must be unfinished at first
        """

        workflowName = 'Repack_Run481516_StreamZ'
        mergeTasks = ['RepackMergewrite_QuadElectron_RAW', 'RepackMergewrite_TriPhoton_RAW',
                      'RepackMergewrite_SingleNeutrino_RAW']

        self.stateMap = {'Merge' : [],
                         'Processing Done' : []}
        self.orderedStates = ['Merge', 'Processing Done']

        # Populate WMStats
        self.wmstatsWriter.insertGenericRequest({'_id' : workflowName})
        self.wmstatsWriter.updateRequestStatus(workflowName, 'Closed')

        # Create a wmspec in disk
        workload = newWorkload(workflowName)
        repackTask = workload.newTask('Repack')
        for task in mergeTasks:
            repackTask.addTask(task)
        repackTask.addTask('RepackCleanupUnmergedwrite_QuadElectron_RAW')

        specPath = os.path.join(self.testDir, 'Repack.pkl')
        workload.save(specPath)

        # Populate WMBS
        topFileset = Fileset(name = 'TestStreamerFileset')
        topFileset.create()

        options = {'spec' : specPath, 'owner' : 'ItsAMeMario',
                   'name' : workflowName, 'wfType' : 'tier0'}
        topLevelWorkflow = Workflow(task = '/%s/Repack' % workflowName,
                                    **options)
        topLevelWorkflow.create()
        topLevelSub = Subscription(topFileset, topLevelWorkflow)
        topLevelSub.create()
        self.stateMap['Merge'].append(topFileset)
        for task in mergeTasks:
            mergeWorkflow = Workflow(task = '/%s/Repack/%s' % (workflowName, task), **options)
            mergeWorkflow.create()
            unmergedFileset = Fileset(name = 'TestUnmergedFileset%s' % task)
            unmergedFileset.create()
            mergeSub = Subscription(unmergedFileset, mergeWorkflow)
            mergeSub.create()
            self.stateMap['Processing Done'].append(unmergedFileset)
        cleanupWorkflow = Workflow(task = '/Repack_Run481516_StreamZ/Repack/RepackCleanupUnmergedwrite_QuadElectron_RAW',
                                   **options)
        cleanupWorkflow.create()
        unmergedFileset = Fileset(name = 'TestUnmergedFilesetToCleanup')
        unmergedFileset.create()
        cleanupSub = Subscription(unmergedFileset, cleanupWorkflow)
        cleanupSub.create()

        return

    def setupExpressWorkflow(self):
        """
        _setupExpressWorkflow_

#.........这里部分代码省略.........

# 需要导入模块: from WMCore.Services.WMStats.WMStatsWriter import WMStatsWriter [as 别名]
# 或者: from WMCore.Services.WMStats.WMStatsWriter.WMStatsWriter import updateRequestStatus [as 别名]
class CleanCouchPoller(BaseWorkerThread):
    """
    Cleans up local couch db according the the given condition.
    1. Cleans local couch db when request is completed and reported to cental db.
       This will clean up local couchdb, local summary db, local queue
       
    2. Cleans old couchdoc which is created older than the time threshold
    
    """
    def __init__(self, config):
        """
        Initialize config
        """
        BaseWorkerThread.__init__(self)
        # set the workqueue service for REST call
        self.config = config
        
    def setup(self, parameters):
        """
        Called at startup
        """
        # set the connection for local couchDB call
        self.useReqMgrForCompletionCheck   = getattr(self.config.TaskArchiver, 'useReqMgrForCompletionCheck', True)
        self.wmstatsCouchDB = WMStatsWriter(self.config.TaskArchiver.localWMStatsURL)
        self.centralCouchDBReader = WMStatsReader(self.config.TaskArchiver.centralWMStatsURL)
        
        if self.useReqMgrForCompletionCheck:
            self.deletableStates = ["announced"]
            self.centralCouchDBWriter = WMStatsWriter(self.config.TaskArchiver.centralWMStatsURL)
        else:
            # Tier0 case
            self.deletableStates = ["completed"]
            self.centralCouchDBWriter = self.wmstatsCouchDB
        
        jobDBurl = sanitizeURL(self.config.JobStateMachine.couchurl)['url']
        jobDBName = self.config.JobStateMachine.couchDBName
        self.jobCouchdb  = CouchServer(jobDBurl)
        self.jobsdatabase = self.jobCouchdb.connectDatabase("%s/jobs" % jobDBName)
        self.fwjrdatabase = self.jobCouchdb.connectDatabase("%s/fwjrs" % jobDBName)

    def algorithm(self, parameters):
        """
        get information from wmbs, workqueue and local couch
        """
        try:
            logging.info("Cleaning up the old request docs")
            report = self.wmstatsCouchDB.deleteOldDocs(self.config.TaskArchiver.DataKeepDays)
            logging.info("%s docs deleted" % report)
            logging.info("getting complete and announced requests")
            
            deletableWorkflows = self.centralCouchDBReader.workflowsByStatus(self.deletableStates)
            
            logging.info("Ready to delete %s" % deletableWorkflows)     
            for workflowName in deletableWorkflows:
                if self.cleanAllLocalCouchDB(workflowName):
                    self.centralCouchDBWriter.updateRequestStatus(workflowName, "normal-archived")
                    logging.info("status updated to normal-archived %s" % workflowName)
            
            abortedWorkflows = self.centralCouchDBReader.workflowsByStatus(["aborted-completed"])
            logging.info("Ready to delete aborted %s" % abortedWorkflows)
            for workflowName in abortedWorkflows:
                if self.cleanAllLocalCouchDB(workflowName):
                    self.centralCouchDBWriter.updateRequestStatus(workflowName, "aborted-archived")
                    logging.info("status updated to aborted-archived %s" % workflowName)

            #TODO: following code is temproraly - remove after production archived data is cleaned 
            removableWorkflows = self.centralCouchDBReader.workflowsByStatus(["archived"])
            
            logging.info("Ready to delete %s from wmagent_summary" % removableWorkflows)     
            for workflowName in removableWorkflows:
                logging.info("Deleting %s from WMAgent Summary Couch" % workflowName)
                report = self.deleteWorkflowFromJobCouch(workflowName, "WMStats")
                logging.info("%s docs deleted from wmagent_summary" % report)
                # only updatet he status when delete is successful
                # TODO: need to handle the case when there are multiple agent running the same request.
                if report["status"] == "ok":
                    self.centralCouchDBWriter.updateRequestStatus(workflowName, "normal-archived")
                    logging.info("status updated to normal-archived from archived (this is temp solution for production) %s" % workflowName)

        except Exception, ex:
            logging.error(str(ex))
            logging.error("Error occurred, will try again next cycle")

# 需要导入模块: from WMCore.Services.WMStats.WMStatsWriter import WMStatsWriter [as 别名]
# 或者: from WMCore.Services.WMStats.WMStatsWriter.WMStatsWriter import updateRequestStatus [as 别名]

#.........这里部分代码省略.........

    def closeOutRealTimeWorkflows(self):
        """
        _closeOutRealTimeWorkflows_

        Updates couch with the closeout status of Repack and Express
        PromptReco should be closed out automatically

        """
        getNotClosedOutWorkflowsDAO = self.daoFactory(classname = "Tier0Feeder.GetNotClosedOutWorkflows")
        workflows = getNotClosedOutWorkflowsDAO.execute()

        if len(workflows) == 0:
            logging.debug("No workflows to publish to couch monitoring, doing nothing")

        if workflows:
            for workflow in workflows:
                (workflowId, filesetId, filesetOpen, workflowName) = workflow
                # find returns -1 if the string is not found
                if workflowName.find('PromptReco') >= 0:
                    logging.debug("Closing out instantaneously PromptReco Workflow %s" % workflowName)
                    self.updateClosedState(workflowName, workflowId)
                else :
                    # Check if fileset (which you already know) is closed or not
                    # FIXME: No better way to do it? what comes from the DAO is a string, casting bool or int doesn't help much.
                    # Works like that :
                    if filesetOpen == '0':
                        self.updateClosedState(workflowName, workflowId)

        return

    def updateClosedState(self, workflowName, workflowId):
        """
        _updateClosedState_

        Mark a workflow as Closed

        """
        markCloseoutWorkflowMonitoringDAO = self.daoFactory(classname = "Tier0Feeder.MarkCloseoutWorkflowMonitoring")

        response = self.localSummaryCouchDB.updateRequestStatus(workflowName, 'Closed')

        if response == "OK" or "EXISTS":
            logging.debug("Successfully closed workflow %s" % workflowName)
            markCloseoutWorkflowMonitoringDAO.execute(workflowId)

        return

    def notifyStorageManager(self):
        """
        _notifyStorageManager_

        Find all finished streamers for closed all run/stream
        Send the notification message to StorageManager
        Update the streamer status to finished (deleted = 1)

        """
        getFinishedStreamersDAO = self.daoFactory(classname = "SMNotification.GetFinishedStreamers")
        markStreamersFinishedDAO = self.daoFactory(classname = "SMNotification.MarkStreamersFinished")

        allFinishedStreamers = getFinishedStreamersDAO.execute(transaction = False)

        num = len(allFinishedStreamers)/50
        for finishedStreamers in [allFinishedStreamers[i::num] for i in range(num)]:

            streamers = []
            filenameParams = ""

            for (id, lfn) in finishedStreamers:
                streamers.append(id)
                filenameParams += "-FILENAME %s " % os.path.basename(lfn)

            logging.debug("Notifying transfer system about processed streamers")
            p = subprocess.Popen("/bin/bash",stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
            output, error = p.communicate("""
            export T0_BASE_DIR=%s
            export T0ROOT=${T0_BASE_DIR}/T0
            export CONFIG=${T0_BASE_DIR}/Config/TransferSystem_CERN.cfg
 
            export PERL5LIB=${T0ROOT}/perl_lib
 
            ${T0ROOT}/operations/sendRepackedStatus.pl --config $CONFIG %s
            """ % (self.transferSystemBaseDir, filenameParams))

            if len(error) > 0:
                logging.error("ERROR: Could not notify transfer system about processed streamers")
                logging.error("ERROR: %s" % error)

            markStreamersFinishedDAO.execute(streamers, transaction = False)

        return

    def terminate(self, params):
        """
        _terminate_

        Kill the code after one final pass when called by the master thread.

        """
        logging.debug("terminating immediately")

# 需要导入模块: from WMCore.Services.WMStats.WMStatsWriter import WMStatsWriter [as 别名]
# 或者: from WMCore.Services.WMStats.WMStatsWriter.WMStatsWriter import updateRequestStatus [as 别名]
class CleanCouchPoller(BaseWorkerThread):
    """
    Cleans up local couch db according the the given condition.
    1. Cleans local couch db when request is completed and reported to cental db.
       This will clean up local couchdb, local summary db, local queue
       
    2. Cleans old couchdoc which is created older than the time threshold
    
    """
    def __init__(self, config):
        """
        Initialize config
        """
        BaseWorkerThread.__init__(self)
        # set the workqueue service for REST call
        self.config = config
        
    def setup(self, parameters):
        """
        Called at startup
        """
        # set the connection for local couchDB call
        self.useReqMgrForCompletionCheck   = getattr(self.config.TaskArchiver, 'useReqMgrForCompletionCheck', True)
        self.wmstatsCouchDB = WMStatsWriter(self.config.TaskArchiver.localWMStatsURL)
        self.centralCouchDBWriter = WMStatsWriter(self.config.TaskArchiver.centralWMStatsURL)
        self.centralCouchDBReader = WMStatsReader(self.config.TaskArchiver.centralWMStatsURL)
        jobDBurl = sanitizeURL(self.config.JobStateMachine.couchurl)['url']
        jobDBName = self.config.JobStateMachine.couchDBName
        self.jobCouchdb  = CouchServer(jobDBurl)
        self.jobsdatabase = self.jobCouchdb.connectDatabase("%s/jobs" % jobDBName)
        self.fwjrdatabase = self.jobCouchdb.connectDatabase("%s/fwjrs" % jobDBName)

    def algorithm(self, parameters):
        """
        get information from wmbs, workqueue and local couch
        """
        try:
            logging.info("Cleaning up the old request docs")
            report = self.wmstatsCouchDB.deleteOldDocs(self.config.TaskArchiver.DataKeepDays)
            logging.info("%s docs deleted" % report)
            logging.info("getting complete and announced requests")
            
            #TODO: define what is deletable status. Also add the code to delet summary document, 
            # request summary and job summary
            if self.useReqMgrForCompletionCheck:
                deletableWorkflows = self.centralCouchDBReader.workflowsByStatus(["announced"])
            else:
                deletableWorkflows = self.centralCouchDBReader.workflowsByStatus(["completed"])
            
            logging.info("Ready to delete %s" % deletableWorkflows)     
            for workflowName in deletableWorkflows:
                logging.info("Deleting %s from JobCouch" % workflowName)
                
                report = self.deleteWorkflowFromJobCouch(workflowName, "JobDump")
                logging.info("%s docs deleted from JobDump" % report)
                report = self.deleteWorkflowFromJobCouch(workflowName, "FWJRDump")
                logging.info("%s docs deleted from FWJRDump" % report)
                
                self.centralCouchDBWriter.updateRequestStatus(workflowName, "archived")
                logging.info("status updated to archived %s" % workflowName)
                
        except Exception, ex:
            logging.error(str(ex))
            logging.error("Error occurred, will try again next cycle")

# 需要导入模块: from WMCore.Services.WMStats.WMStatsWriter import WMStatsWriter [as 别名]
# 或者: from WMCore.Services.WMStats.WMStatsWriter.WMStatsWriter import updateRequestStatus [as 别名]
def updateRequestStatus(couchURL, requestList, status):
    ww = WMStatsWriter(couchURL)
    for request in requestList:
        ww.updateRequestStatus(request, status)
        print("%s is udated to %s" % (request, status))