Python BossAirAPI.getComplete方法代码示例

# 需要导入模块: from WMCore.BossAir.BossAirAPI import BossAirAPI [as 别名]
# 或者: from WMCore.BossAir.BossAirAPI.BossAirAPI import getComplete [as 别名]
    def testD_PrototypeChain(self):
        """
        _PrototypeChain_

        Prototype the BossAir workflow
        """
        myThread = threading.currentThread()

        nRunning = getCondorRunningJobs(self.user)
        self.assertEqual(nRunning, 0, "User currently has %i running jobs.  Test will not continue" % (nRunning))


        config = self.getConfig()
        config.BossAir.pluginName = 'CondorPlugin'

        baAPI  = BossAirAPI(config = config)

        workload = self.createTestWorkload()

        workloadName = "basicWorkload"

        changeState = ChangeState(config)

        nSubs = 5
        nJobs = 10

        cacheDir = os.path.join(self.testDir, 'CacheDir')

        jobGroupList = self.createJobGroups(nSubs = nSubs, nJobs = nJobs,
                                            task = workload.getTask("ReReco"),
                                            workloadSpec = os.path.join(self.testDir,
                                                                        'workloadTest',
                                                                        workloadName),
                                            site = 'se.T2_US_UCSD')
        for group in jobGroupList:
            changeState.propagate(group.jobs, 'created', 'new')


        jobSubmitter = JobSubmitterPoller(config = config)
        jobTracker   = JobTrackerPoller(config = config)
        statusPoller = StatusPoller(config = config)

        jobSubmitter.algorithm()

        nRunning = getCondorRunningJobs(self.user)
        self.assertEqual(nRunning, nSubs * nJobs)

        newJobs = baAPI._loadByStatus(status = 'New')
        self.assertEqual(len(newJobs), nSubs * nJobs)

        # Check WMBS
        getJobsAction = self.daoFactory(classname = "Jobs.GetAllJobs")
        result = getJobsAction.execute(state = 'Executing', jobType = "Processing")
        self.assertEqual(len(result), nSubs * nJobs)

        statusPoller.algorithm()

        nRunning = getCondorRunningJobs(self.user)
        self.assertEqual(nRunning, nSubs * nJobs)

        newJobs = baAPI._loadByStatus(status = 'New')
        self.assertEqual(len(newJobs), 0)

        newJobs = baAPI._loadByStatus(status = 'Idle')
        self.assertEqual(len(newJobs), nSubs * nJobs)


        # Tracker should do nothing
        jobTracker.algorithm()

        result = getJobsAction.execute(state = 'Executing', jobType = "Processing")
        self.assertEqual(len(result), nSubs * nJobs)


        # Wait for jobs to timeout due to short Pending wait period
        time.sleep(12)


        statusPoller.algorithm()

        newJobs = baAPI._loadByStatus(status = 'Idle')
        self.assertEqual(len(newJobs), 0)

        newJobs = baAPI._loadByStatus(status = 'Timeout', complete = '0')
        self.assertEqual(len(newJobs), nSubs * nJobs)

        # Jobs should be gone
        nRunning = getCondorRunningJobs(self.user)
        self.assertEqual(nRunning, 0)


        # Check if they're complete
        completeJobs = baAPI.getComplete()
        self.assertEqual(len(completeJobs), nSubs * nJobs)


        # Because they timed out, they all should have failed
        jobTracker.algorithm()

        result = getJobsAction.execute(state = 'Executing', jobType = "Processing")
#.........这里部分代码省略.........

# 需要导入模块: from WMCore.BossAir.BossAirAPI import BossAirAPI [as 别名]
# 或者: from WMCore.BossAir.BossAirAPI.BossAirAPI import getComplete [as 别名]
class JobTrackerPoller(BaseWorkerThread):
    """
    _JobTrackerPoller_

    Polls the BossAir database for complete jobs
    Handles completed jobs
    """

    def __init__(self, config):
        """
        Initialise class members
        """

        BaseWorkerThread.__init__(self)
        self.config = config

        myThread = threading.currentThread()

        self.changeState = ChangeState(self.config)
        self.bossAir = BossAirAPI(config=config)
        self.daoFactory = DAOFactory(package="WMCore.WMBS", logger=myThread.logger, dbinterface=myThread.dbi)

        self.jobListAction = self.daoFactory(classname="Jobs.GetAllJobs")

        # initialize the alert framework (if available)
        self.initAlerts(compName="JobTracker")

    def setup(self, parameters=None):
        """
        Load DB objects required for queries
        """

        return

    def terminate(self, params=None):
        """
        _terminate_

        Terminate the function after one more run.
        """
        logging.debug("terminating. doing one more pass before we die")
        self.algorithm(params)
        return

    def algorithm(self, parameters=None):
        """
        Performs the archiveJobs method, looking for each type of failure
        And deal with it as desired.
        """
        logging.info("Running Tracker algorithm")
        myThread = threading.currentThread()
        try:
            self.trackJobs()
        except WMException as ex:
            if getattr(myThread, "transaction", None):
                myThread.transaction.rollback()
            self.sendAlert(6, msg=str(ex))
            raise
        except Exception as ex:
            msg = "Unknown exception in JobTracker!\n"
            msg += str(ex)
            if getattr(myThread, "transaction", None):
                myThread.transaction.rollback()
            self.sendAlert(6, msg=msg)
            logging.error(msg)
            raise JobTrackerException(msg)

        return

    def trackJobs(self):
        """
        _trackJobs_

        Finds a list of running jobs and the sites that they're running at,
        and passes that off to tracking.
        """

        passedJobs = []
        failedJobs = []

        # Get all jobs WMBS thinks are running
        jobList = self.jobListAction.execute(state="Executing")

        if jobList == []:
            # No jobs: do nothing
            return

        logging.info("Have list of %i executing jobs" % len(jobList))

        # Now get all jobs that BossAir thinks are complete
        completeJobs = self.bossAir.getComplete()

        logging.info("%i jobs are complete in BossAir" % (len(completeJobs)))
        logging.debug(completeJobs)

        for job in completeJobs:
            if not job["id"] in jobList:
                continue
            if job["status"].lower() == "timeout":
                failedJobs.append(job)
#.........这里部分代码省略.........

# 需要导入模块: from WMCore.BossAir.BossAirAPI import BossAirAPI [as 别名]
# 或者: from WMCore.BossAir.BossAirAPI.BossAirAPI import getComplete [as 别名]
    def testB_PluginTest(self):
        """
        _PluginTest_


        Now check that these functions worked if called through plugins
        Instead of directly.

        There are only three plugin
        """
        #return

        myThread = threading.currentThread()

        config = self.getConfig()

        baAPI  = BossAirAPI(config = config)


        # Create some jobs
        nJobs = 10

        jobDummies = self.createDummyJobs(nJobs = nJobs, location = 'Xanadu')
        changeState = ChangeState(config)
        changeState.propagate(jobDummies, 'created', 'new')
        changeState.propagate(jobDummies, 'executing', 'created')

        # Prior to building the job, each job must have a plugin
        # and user assigned
        for job in jobDummies:
            job['plugin']   = 'TestPlugin'
            job['owner']    = 'tapas'

        baAPI.submit(jobs = jobDummies)


        newJobs = baAPI._loadByStatus(status = 'New')
        self.assertEqual(len(newJobs), nJobs)

        # Should be no more running jobs
        runningJobs = baAPI._listRunJobs()
        self.assertEqual(len(runningJobs), nJobs)


        # Test Plugin should complete all jobs
        baAPI.track()

        # Should be no more running jobs
        runningJobs = baAPI._listRunJobs()
        self.assertEqual(len(runningJobs), 0)


        # Check if they're complete
        completeJobs = baAPI.getComplete()
        self.assertEqual(len(completeJobs), nJobs)


        # Do this test because BossAir is specifically built
        # to keep it from finding completed jobs
        result = myThread.dbi.processData("SELECT id FROM bl_runjob")[0].fetchall()
        self.assertEqual(len(result), nJobs)


        baAPI.removeComplete(jobs = jobDummies)


        result = myThread.dbi.processData("SELECT id FROM bl_runjob")[0].fetchall()
        self.assertEqual(len(result), 0)


        return

# 需要导入模块: from WMCore.BossAir.BossAirAPI import BossAirAPI [as 别名]
# 或者: from WMCore.BossAir.BossAirAPI.BossAirAPI import getComplete [as 别名]
class JobTrackerPoller(BaseWorkerThread):
    """
    _JobTrackerPoller_

    Polls the BossAir database for complete jobs
    Handles completed jobs
    """

    def __init__(self, config):
        """
        Initialise class members
        """

        BaseWorkerThread.__init__(self)
        self.config = config

        myThread = threading.currentThread()
        self.changeState = ChangeState(self.config)
        self.bossAir = BossAirAPI(config=config)
        self.daoFactory = DAOFactory(package="WMCore.WMBS",
                                     logger=myThread.logger,
                                     dbinterface=myThread.dbi)

        self.jobListAction = self.daoFactory(classname="Jobs.GetAllJobs")
        self.setFWJRAction = self.daoFactory(classname="Jobs.SetFWJRPath")

    def setup(self, parameters=None):
        """
        Load DB objects required for queries
        """

        return

    def terminate(self, params=None):
        """
        _terminate_

        Terminate the function after one more run.
        """
        logging.debug("terminating. doing one more pass before we die")
        self.algorithm(params)
        return

    def algorithm(self, parameters=None):
        """
        Performs the archiveJobs method, looking for each type of failure
        And deal with it as desired.
        """
        logging.info("Running Tracker algorithm")
        myThread = threading.currentThread()
        try:
            self.trackJobs()
        except WMException:
            if getattr(myThread, 'transaction', None):
                myThread.transaction.rollback()
            raise
        except Exception as ex:
            msg = "Unknown exception in JobTracker!\n"
            msg += str(ex)
            if getattr(myThread, 'transaction', None):
                myThread.transaction.rollback()
            logging.error(msg)
            raise JobTrackerException(msg)

        return

    def trackJobs(self):
        """
        _trackJobs_

        Finds a list of running jobs and the sites that they're running at,
        and passes that off to tracking.
        """
        passedJobs = []
        failedJobs = []

        jobList = self.jobListAction.execute(state="executing")
        logging.info("Have list of %i executing jobs in WMBS", len(jobList))

        if not jobList:
            return

        # retrieve completed jobs from BossAir that are 'executing' in WMBS
        completeJobs = self.bossAir.getComplete()
        logging.info("Have list of %i jobs complete in BossAir but executing in WMBS", len(completeJobs))
        logging.debug(completeJobs)

        for job in completeJobs:
            if job['id'] not in jobList:
                logging.error("Found a complete job in BossAir without a correspondent in WMBS!")
                continue
            if job['status'].lower() == 'timeout':
                failedJobs.append(job)
            else:
                passedJobs.append(job)

        # Assume all these jobs "passed" if they aren't in timeout
        self.passJobs(passedJobs)
        self.failJobs(failedJobs)

#.........这里部分代码省略.........