Python RequestManager.RequestManager類代碼示例

def getSiteInfoFromReqMgr(serviceURL):
    """ get agent info from request mgr """

    reqMgr = RequestManager({'endpoint':serviceURL})
    #get information from global queue.
    try:
        queues = reqMgr.getWorkQueue()
    except Exception, ex:
        errorInfo = {}
        errorInfo['site_name'] = serviceURL
        return [errorInfo]

def getSplitting(requestName):
	reqmgrUrl='https://cmsweb.cern.ch/reqmgr/reqMgr/'
	reqmgr = RequestManager(dict = {'endpoint' : reqmgrUrl})
	result = reqmgr.getRequest(requestName)
	workloadDB = Database(result['CouchWorkloadDBName'], result['CouchURL'])
	workloadPickle = workloadDB.getAttachment(requestName, 'spec')
	spec = pickle.loads(workloadPickle)
	workload = WMWorkloadHelper(spec)
	params = workload.getTopLevelTask()[0].jobSplittingParameters()
	algo = workload.getTopLevelTask()[0].jobSplittingAlgorithm()
	return params['events_per_job']

def main():
    demPolicy = Block()
    reqmgr = RequestManager(dict = {'endpoint' : 'https://cmsweb.cern.ch/reqmgr/reqMgr'})
    result = reqmgr.getRequest('pdmvserv_HIG-Summer12DR53X-01392_T1_ES_PIC_MSS_1_v0__130724_063344_7207')
    workloadDB = Database(result['CouchWorkloadDBName'], result['CouchURL'])
    workloadPickle = workloadDB.getAttachment('pdmvserv_HIG-Summer12DR53X-01392_T1_ES_PIC_MSS_1_v0__130724_063344_7207', 'spec')
    spec = pickle.loads(workloadPickle)
    workload = WMWorkloadHelper(spec)
    x,y = demPolicy(wmspec = workload, task = workload.getTopLevelTask()[0])
    print x
    print y

def getAgentInfoFromReqMgr(serviceURL):
    """ get agent info from request mgr """

    reqMgr = RequestManager({'endpoint':serviceURL})
    #get information from global queue.
    try:
        gQueues = reqMgr.getWorkQueue()
    except Exception, ex:
        errorInfo = {}
        errorInfo['url'] = serviceURL
        errorInfo['status'] = "Request Manager down: %s" % serviceURL
        errorInfo['acdc'] = 'N/A'
        return [errorInfo]

def getRequestInformationAndWorkload(requestName, reqmgrUrl):
    """
    _getRequestInformationAndWorkload_

    Retrieve the request information for assignment
    and the full pickled workload.
    """
    reqmgr = RequestManager(dict = {'endpoint' : reqmgrUrl})
    result = reqmgr.getRequest(requestName)
    workloadDB = Database(result['CouchWorkloadDBName'], result['CouchURL'])
    workloadPickle = workloadDB.getAttachment(requestName, 'spec')
    spec = pickle.loads(workloadPickle)
    workload = WMWorkloadHelper(spec)
    return workload, result

def getAgentInfoFromReqMgr(serviceURL):
    """ get agent info from request mgr """
    reqMgr = RequestManager({'endpoint':serviceURL})
    try:
        if serviceURL.lower() == "local":
            gQueues = getGlobalQueues()
        else:
            gQueues = reqMgr.getWorkQueue()
    except Exception, ex:
        logging.warning("Error: %s" % str(ex))
        errorInfo = {}
        errorInfo['url'] = serviceURL
        errorInfo['status'] = "Request Manager down: %s" % serviceURL
        errorInfo['acdc'] = 'N/A'
        return [errorInfo]

    def setUp(self):
        RESTBaseUnitTest.setUp(self)
        self.testInit.setupCouch("%s" % self.couchDBName, "GroupUser", "ConfigCache", "ReqMgr")
        self.testInit.setupCouch("%s_wmstats" % self.couchDBName, "WMStats")
        # logging stuff from TestInit is broken, setting myself
        l = logging.getLogger()
        l.setLevel(logging.DEBUG)
        self.params = {}
        self.params['endpoint'] = self.config.getServerUrl()
        self.reqService = RequestManagerDS(self.params)
        self.jsonSender = JSONRequests(self.config.getServerUrl())

        userName     = 'Taizong'
        groupName    = 'Li'
        teamName     = 'Tang'
        schema = utils.getAndSetupSchema(self,
                                         userName = userName,
                                         groupName = groupName,
                                         teamName = teamName)
        schema['ConfigCacheID'] = self.createConfig()
        schema['CouchDBName'] = self.couchDBName
        schema['CouchWorkloadDBName'] = self.couchDBName

        try:
            r = self.jsonSender.put('request', schema)
            try:
                self.requestName = r[0]['RequestName']
            except:
                self.requestName = r[0].values()[0]['RequestName']
        except Exception as ex:
            msg = traceback.format_exc()
            print("Exception during set up, reason: %s" % msg)
            raise ex

def getRequestInfoFromReqMgr(serviceURL):
    """ get the request info from requestManager """

    ###TODO: add back when GlobalMonitor spins out as a separate application
    service = RequestManager({'endpoint':serviceURL})
    try:
        ### use request manager funtion directly
        ### TODO: remove this when GlobalMonitor spins out as a separate application
        if serviceURL.lower() == "local":
            
            baseResults = getOverview()
            urls = getGlobalQueues()
        else:
            baseResults = service.getRequestNames()
            urls = service.getWorkQueue()
    except Exception, ex:
        logging.error(str(ex))
        return DFormatter.errorFormatter(serviceURL, "RequestManger Down")

    def __init__(self, config):
        """
        Initialise class members
        """
        BaseWorkerThread.__init__(self)

        myThread = threading.currentThread()
        self.daoFactory = DAOFactory(package="WMCore.WMBS", logger=myThread.logger, dbinterface=myThread.dbi)

        self.dbsDaoFactory = DAOFactory(
            package="WMComponent.DBS3Buffer", logger=myThread.logger, dbinterface=myThread.dbi
        )

        self.config = config
        self.jobCacheDir = self.config.JobCreator.jobCacheDir

        if getattr(self.config.TaskArchiver, "useWorkQueue", False) != False:
            # Get workqueue setup from config unless overridden
            if hasattr(self.config.TaskArchiver, "WorkQueueParams"):
                self.workQueue = localQueue(**self.config.TaskArchiver.WorkQueueParams)
            else:
                from WMCore.WorkQueue.WorkQueueUtils import queueFromConfig

                self.workQueue = queueFromConfig(self.config)
        else:
            self.workQueue = None

        self.timeout = getattr(self.config.TaskArchiver, "timeOut", None)
        self.useReqMgrForCompletionCheck = getattr(self.config.TaskArchiver, "useReqMgrForCompletionCheck", True)

        if not self.useReqMgrForCompletionCheck:
            # sets the local monitor summary couch db
            self.requestLocalCouchDB = RequestDBWriter(
                self.config.AnalyticsDataCollector.localT0RequestDBURL,
                couchapp=self.config.AnalyticsDataCollector.RequestCouchApp,
            )
            self.centralCouchDBWriter = self.requestLocalCouchDB
        else:
            self.centralCouchDBWriter = RequestDBWriter(self.config.AnalyticsDataCollector.centralRequestDBURL)

            self.reqmgr2Svc = ReqMgr(self.config.TaskArchiver.ReqMgr2ServiceURL)
            # TODO: remove this when reqmgr2 replace reqmgr completely (reqmgr2Only)
            self.reqmgrSvc = RequestManager({"endpoint": self.config.TaskArchiver.ReqMgrServiceURL})

        # Load the cleanout state ID and save it
        stateIDDAO = self.daoFactory(classname="Jobs.GetStateID")
        self.stateID = stateIDDAO.execute("cleanout")

        return

 def setUp(self):
     """
     setUP global values
     """
     RESTBaseUnitTest.setUp(self)
     self.params = {}
     self.params['endpoint'] = self.config.getServerUrl()
     self.reqService = RequestManagerDS(self.params)
     self.jsonSender = JSONRequests(self.config.getServerUrl())
     self.requestSchema = getRequestSchema()
     self.jsonSender.put('group/PeopleLikeMe')
     self.jsonSender.put('user/[email protected]')
     self.jsonSender.put('group/PeopleLikeMe/me')
     self.jsonSender.put('version/CMSSW_3_5_8')
     self.jsonSender.put('request/' + self.requestSchema['RequestName'], 
                         self.requestSchema)

    def setUp(self):
        """
        setUP global values
        Database setUp is done in base class
        """
        self.couchDBName = "reqmgr_t_0"
        RESTBaseUnitTest.setUp(self)
        self.testInit.setupCouch("%s" % self.couchDBName, "ConfigCache", "ReqMgr")
        self.testInit.setupCouch("%s_wmstats" % self.couchDBName,
                                 "WMStats")
        self.testInit.setupCouch("%s_acdc" % self.couchDBName,
                                 "ACDC", "GroupUser")
        reqMgrHost = self.config.getServerUrl()
        self.jsonSender = JSONRequests(reqMgrHost)

        self.params = {}
        self.params['endpoint'] = reqMgrHost
        self.reqService = RequestManager(self.params)

 def __init__(self, **kwargs):
     if not kwargs.get('logger'):
         import logging
         kwargs['logger'] = logging
     self.logger = kwargs['logger']
     self.reqMgr = RequestManager(kwargs)
     self.reqmgr2Only = kwargs.get("reqmgr2_only", False)
     #this will break all in one test
     self.reqMgr2 = ReqMgr(kwargs.get("reqmgr2_endpoint", None))
     
     centralurl = kwargs.get("central_logdb_url", "")
     identifier = kwargs.get("log_reporter", "")
     
     # set the thread name before creat the log db. 
     # only sets that when it is not set already
     myThread = threading.currentThread()
     if myThread.getName() == "MainThread":
         myThread.setName(self.__class__.__name__)
         
     self.logdb = LogDB(centralurl, identifier, logger=self.logger)
     self.previous_state = {}

    def __init__(self, config):
        """
        __init__
        """
        BaseWorkerThread.__init__(self)
        self.config = config

        self.bossAir = BossAirAPI(config = self.config)
        self.reqmgr = RequestManager({'endpoint' : self.config.JobUpdater.reqMgrUrl})
        self.workqueue = WorkQueue(self.config.WorkQueueManager.couchurl,
                                   self.config.WorkQueueManager.dbname)

        myThread = threading.currentThread()

        self.daoFactory = DAOFactory(package = "WMCore.WMBS",
                                     logger = myThread.logger,
                                     dbinterface = myThread.dbi)

        self.listWorkflowsDAO = self.daoFactory(classname = "Workflow.ListForJobUpdater")
        self.updateWorkflowPrioDAO = self.daoFactory(classname = "Workflow.UpdatePriority")
        self.executingJobsDAO = self.daoFactory(classname = "Jobs.GetNumberOfJobsForWorkflowTaskStatus")

 def setup(self, parameters):
     """
     Called at startup
     """
     self.teamName = self.config.Agent.teamName
     # set the connection for local couchDB call
     self.useReqMgrForCompletionCheck = getattr(self.config.TaskArchiver, 'useReqMgrForCompletionCheck', True)
     self.archiveDelayHours = getattr(self.config.TaskArchiver, 'archiveDelayHours', 0)
     self.wmstatsCouchDB = WMStatsWriter(self.config.TaskArchiver.localWMStatsURL, 
                                         "WMStatsAgent")
     
     #TODO: we might need to use local db for Tier0
     self.centralRequestDBReader = RequestDBReader(self.config.AnalyticsDataCollector.centralRequestDBURL, 
                                                   couchapp=self.config.AnalyticsDataCollector.RequestCouchApp)
     
     if self.useReqMgrForCompletionCheck:
         self.deletableState = "announced"
         self.centralRequestDBWriter = RequestDBWriter(self.config.AnalyticsDataCollector.centralRequestDBURL, 
                                                       couchapp=self.config.AnalyticsDataCollector.RequestCouchApp)
         if self.config.TaskArchiver.reqmgr2Only:
             self.reqmgr2Svc = ReqMgr(self.config.TaskArchiver.ReqMgr2ServiceURL)
         else:
             #TODO: remove this for reqmgr2
             self.reqmgrSvc = RequestManager({'endpoint': self.config.TaskArchiver.ReqMgrServiceURL})
     else:
         # Tier0 case
         self.deletableState = "completed"
         # use local for update
         self.centralRequestDBWriter = RequestDBWriter(self.config.AnalyticsDataCollector.localT0RequestDBURL, 
                                                       couchapp=self.config.AnalyticsDataCollector.RequestCouchApp)
     
     jobDBurl = sanitizeURL(self.config.JobStateMachine.couchurl)['url']
     jobDBName = self.config.JobStateMachine.couchDBName
     self.jobCouchdb  = CouchServer(jobDBurl)
     self.jobsdatabase = self.jobCouchdb.connectDatabase("%s/jobs" % jobDBName)
     self.fwjrdatabase = self.jobCouchdb.connectDatabase("%s/fwjrs" % jobDBName)
     
     statSummaryDBName = self.config.JobStateMachine.summaryStatsDBName
     self.statsumdatabase = self.jobCouchdb.connectDatabase(statSummaryDBName)

class RequestManagerTest(RESTBaseUnitTest):
    """
    Test RequestMgr Service client
    It will start RequestMgr RESTService
    Server DB is whatever env is set     
    
    This checks whether DS call makes without error and return the results.
    This test only test service call returns without error.
    The correctness of each function is tested in test/python/RequestManager_t/RequestMgr_t.py
    
    """
    def initialize(self):
        self.couchDBName = "reqmgr_t_0"
        self.config = RequestManagerConfig(
                'WMCore.HTTPFrontEnd.RequestManager.ReqMgrRESTModel')
        dbUrl = os.environ.get("DATABASE", None)
        self.config.setDBUrl(dbUrl)        
        self.config.setFormatter('WMCore.WebTools.RESTFormatter')
        self.config.setupRequestConfig()
        self.config.setupCouchDatabase(dbName = self.couchDBName)
        self.config.setPort(8899)
        self.schemaModules = ["WMCore.RequestManager.RequestDB"]
                
        
    def setUp(self):
        RESTBaseUnitTest.setUp(self)
        self.testInit.setupCouch("%s" % self.couchDBName, "GroupUser", "ConfigCache", "ReqMgr")
        self.testInit.setupCouch("%s_wmstats" % self.couchDBName, "WMStats")
        # logging stuff from TestInit is broken, setting myself
        l = logging.getLogger()
        l.setLevel(logging.DEBUG)
        self.params = {}
        self.params['endpoint'] = self.config.getServerUrl()
        self.reqService = RequestManagerDS(self.params)
        self.jsonSender = JSONRequests(self.config.getServerUrl())

        userName     = 'Taizong'
        groupName    = 'Li'
        teamName     = 'Tang'
        schema = utils.getAndSetupSchema(self,
                                         userName = userName,
                                         groupName = groupName,
                                         teamName = teamName)
        schema['ConfigCacheID'] = self.createConfig()
        schema['CouchDBName'] = self.couchDBName
        schema['CouchWorkloadDBName'] = self.couchDBName

        try:
            r = self.jsonSender.put('request', schema)
            try:
                self.requestName = r[0]['RequestName']
            except:
                self.requestName = r[0].values()[0]['RequestName']
        except Exception as ex:
            msg = traceback.format_exc()
            print("Exception during set up, reason: %s" % msg)
            raise ex

    def tearDown(self):
        self.config.deleteWorkloadCache()
        RESTBaseUnitTest.tearDown(self)
        self.testInit.tearDownCouch()

    def createConfig(self, bad = False):
        """
        _createConfig_

        Create a config of some sort that we can load out of ConfigCache
        """
        PSetTweak = {'process': {'outputModules_': ['ThisIsAName'],
                                 'ThisIsAName': {'dataset': {'dataTier': 'RECO',
                                                             'filterName': 'Filter'}}}}
        BadTweak  = {'process': {'outputModules_': ['ThisIsAName1', 'ThisIsAName2'],
                                 'ThisIsAName1': {'dataset': {'dataTier': 'RECO',
                                                             'filterName': 'Filter'}},
                                 'ThisIsAName2': {'dataset': {'dataTier': 'RECO',
                                                             'filterName': 'Filter'}}}}
        configCache = ConfigCache(os.environ["COUCHURL"], couchDBName = self.couchDBName)
        configCache.createUserGroup(groupname = "testGroup", username = 'testOps')
        if bad:
            configCache.setPSetTweaks(PSetTweak = BadTweak)
        else:
            configCache.setPSetTweaks(PSetTweak = PSetTweak)
        configCache.save()
        return configCache.getCouchID()

    @attr("integration")
    def testA_RequestManagerService(self):
        requestName = self.requestName
        
        request = self.reqService.getRequest(requestName)
        # minimal test : it's return type  and the some value inside
        self.assertEqual(type(request), dict)
        self.assertTrue(len(request) > 0)
        
        # Test putTeam
        self.reqService.putTeam("team_usa")
        self.assertTrue('team_usa' in self.jsonSender.get('team')[0])
        
        self.jsonSender.put('assignment/%s/%s' % ("team_usa", requestName))
#.........這裏部分代碼省略.........