Python DBSBufferUtil.getPhEDExDBSStatusForCompletedWorkflows方法代码示例

# 需要导入模块: from WMComponent.DBS3Buffer.DBSBufferUtil import DBSBufferUtil [as 别名]
# 或者: from WMComponent.DBS3Buffer.DBSBufferUtil.DBSBufferUtil import getPhEDExDBSStatusForCompletedWorkflows [as 别名]
class AnalyticsPoller(BaseWorkerThread):
    """
    Gether the summary data for request (workflow) from local queue,
    local job couchdb, wmbs/boss air and populate summary db for monitoring
    """
    def __init__(self, config):
        """
        initialize properties specified from config
        """
        BaseWorkerThread.__init__(self)
        # set the workqueue service for REST call
        self.config = config
        # need to get campaign, user, owner info
        self.agentInfo = initAgentInfo(self.config)
        self.summaryLevel = (config.AnalyticsDataCollector.summaryLevel).lower()
        self.pluginName = getattr(config.AnalyticsDataCollector, "pluginName", None)
        self.plugin = None

    def setup(self, parameters):
        """
        set db connection(couchdb, wmbs) to prepare to gather information
        """
        # set the connection to local queue
        if not hasattr(self.config, "Tier0Feeder"):
            self.localQueue = WorkQueueService(self.config.AnalyticsDataCollector.localQueueURL)

        # set the connection for local couchDB call
        self.localCouchDB = LocalCouchDBData(self.config.AnalyticsDataCollector.localCouchURL,
                                             self.config.JobStateMachine.summaryStatsDBName,
                                             self.summaryLevel)

        # interface to WMBS/BossAir db
        myThread = threading.currentThread()
        # set wmagent db data
        self.wmagentDB = WMAgentDBData(self.summaryLevel, myThread.dbi, myThread.logger)
        # set the connection for local couchDB call
        self.localSummaryCouchDB = WMStatsWriter(self.config.AnalyticsDataCollector.localWMStatsURL,
                                                 appName="WMStatsAgent")

        # use local db for tier0
        if hasattr(self.config, "Tier0Feeder"):
            centralRequestCouchDBURL = self.config.AnalyticsDataCollector.localT0RequestDBURL
        else:
            centralRequestCouchDBURL = self.config.AnalyticsDataCollector.centralRequestDBURL

        self.centralRequestCouchDB = RequestDBWriter(centralRequestCouchDBURL,
                                                     couchapp=self.config.AnalyticsDataCollector.RequestCouchApp)
        self.centralWMStatsCouchDB = WMStatsWriter(self.config.General.centralWMStatsURL)

        #TODO: change the config to hold couch url
        self.localCouchServer = CouchMonitor(self.config.JobStateMachine.couchurl)

        self.dbsBufferUtil = DBSBufferUtil()

        if self.pluginName is not None:
            pluginFactory = WMFactory("plugins", "WMComponent.AnalyticsDataCollector.Plugins")
            self.plugin = pluginFactory.loadObject(classname=self.pluginName)

    @timeFunction
    def algorithm(self, parameters):
        """
        get information from wmbs, workqueue and local couch
        """
        try:
            # jobs per request info
            logging.info("Getting Job Couch Data ...")
            jobInfoFromCouch = self.localCouchDB.getJobSummaryByWorkflowAndSite()

            # fwjr per request info
            logging.info("Getting FWJRJob Couch Data ...")

            fwjrInfoFromCouch = self.localCouchDB.getJobPerformanceByTaskAndSiteFromSummaryDB()
            skippedInfoFromCouch = self.localCouchDB.getSkippedFilesSummaryByWorkflow()

            logging.info("Getting Batch Job Data ...")
            batchJobInfo = self.wmagentDB.getBatchJobInfo()

            logging.info("Getting Finished Task Data ...")
            finishedTasks = self.wmagentDB.getFinishedSubscriptionByTask()

            logging.info("Getting DBS PhEDEx upload status ...")
            completedWfs = self.dbsBufferUtil.getPhEDExDBSStatusForCompletedWorkflows(summary=True)

            # get the data from local workqueue:
            # request name, input dataset, inWMBS, inQueue
            logging.info("Getting Local Queue Data ...")
            localQInfo = {}
            if not hasattr(self.config, "Tier0Feeder"):
                localQInfo = self.localQueue.getAnalyticsData()
            else:
                logging.debug("Tier-0 instance, not checking WorkQueue")

            # combine all the data from 3 sources
            logging.info("""Combining data from
                                   Job Couch(%s),
                                   FWJR(%s),
                                   WorkflowsWithSkippedFile(%s),
                                   Batch Job(%s),
                                   Finished Tasks(%s),
                                   Local Queue(%s)
#.........这里部分代码省略.........

# 需要导入模块: from WMComponent.DBS3Buffer.DBSBufferUtil import DBSBufferUtil [as 别名]
# 或者: from WMComponent.DBS3Buffer.DBSBufferUtil.DBSBufferUtil import getPhEDExDBSStatusForCompletedWorkflows [as 别名]
class DBSBufferUtilTest(unittest.TestCase):
    def setUp(self):
        """
        _setUp_

        Setup the database and logging connection.  Create some DBSBuffer tables and fake data for testing
        """
        self.testInit = TestInit(__file__)
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()
        self.testInit.setSchema(customModules=["WMComponent.DBS3Buffer", "WMCore.WMBS"],
                                useDefault=False)

        myThread = threading.currentThread()
        self.dbsbufferFactory = DAOFactory(package="WMComponent.DBS3Buffer",
                                     logger=myThread.logger,
                                     dbinterface=myThread.dbi)

        self.wmbsFactory = DAOFactory(package="WMCore.WMBS",
                                           logger=myThread.logger,
                                           dbinterface=myThread.dbi)
        self.dbsUtil = DBSBufferUtil()

        # Create two test dbsbuffer workflows
        insertWorkflow = self.dbsbufferFactory(classname="InsertWorkflow")
        insertWorkflow.execute("Test1", "Task1", 0, 0, 0, 0)
        insertWorkflow.execute("Test2", "Task2", 0, 0, 0, 0)

        # Update one workflow to "completed" state
        updateWorkflow = self.dbsbufferFactory(classname="UpdateWorkflowsToCompleted")
        updateWorkflow.execute(["Test1"])

        # Create a test wmbs workflow
        testWorkflow = Workflow(spec="somespec.xml", owner="Erik", name="Test1", task="Task1")
        testWorkflow.create()

        # Create a test dbsbuffer file
        self.createTestFiles()

    def createTestFiles(self):
        """
        _createTestFiles_

        Create some dbsbuffer test files with different statuses
        :return:
        """
        phedexStatus = self.dbsbufferFactory(classname="DBSBufferFiles.SetPhEDExStatus")

        for i in range(0, 4):

            lfn = "/path/to/some/lfn" + str(i)

            # Two files should be InDBS, two files should be NOTUPLOADED
            if i in [0,2]:
                status = 'InDBS'
            else:
                status = 'NOTUPLOADED'

            testDBSFile = DBSBufferFile(lfn=lfn, size=600000, events=60000, status=status, workflowId=1)

            testDBSFile.setAlgorithm(appName="cmsRun", appVer="UNKNOWN",
                                     appFam="RECO", psetHash="SOMEHASH" + str(i),
                                     configContent="SOMECONTENT")
            testDBSFile.setDatasetPath("/path/to/some/dataset")
            testDBSFile.create()

            # Create all four combinations of status(InDBS,NOTUPLOADED) and in_phedex(0,1)
            if i in [0,1]:
                phedexStatus.execute(lfn, 1)

    def tearDown(self):
        """
        _tearDown_

        Drop all the DBSBuffer tables.
        """
        self.testInit.clearDatabase()

    # List of methods to potentially test
    # def loadDBSBufferFilesBulk(self, fileObjs):
    # def findUploadableDAS(self):
    # def testFindOpenBlocks(self):
    # def loadBlocksByDAS(self, das):
    #
    # def loadBlocks(self, blocknames):
    #
    # def findUploadableFilesByDAS(self, datasetpath):
    #
    # def loadFilesByBlock(self, blockname):

    def testGetPhEDExDBSStatusForCompletedWorkflows(self):
        """
        _testGetPhEDExDBSStatusForCompletedWorkflows_

        :return:
        """

        results = self.dbsUtil.getPhEDExDBSStatusForCompletedWorkflows()
        self.assertEqual(results["Test1"]["InDBS"], 2, "ERROR: Files with InDBS status is incorrect.")
        self.assertEqual(results["Test1"]["InPhEDEx"], 2, "ERROR: Files with InPhEDEx status is incorrect.")
#.........这里部分代码省略.........