Python JSONRequests.get方法代码示例

# 需要导入模块: from WMCore.Services.Requests import JSONRequests [as 别名]
# 或者: from WMCore.Services.Requests.JSONRequests import get [as 别名]
def getRequestInfo():
    requestName = sys.argv[1]
    print "Getting information from cmsweb about %s... %s" % (requestName, time.strftime('%H:%M:%S'))
    requestor = JSONRequests(url = 'https://cmsweb.cern.ch/reqmgr/reqMgr')
    response = requestor.get('/request?requestName=%s' % requestName)
    if response[1] != 200:
        raise RuntimeError("Request information was not available!")
    requestDict = response[0]

    #print requestDict.get('RunWhitelist')
    #print requestDict.get('RunBlacklist')

    if not requestDict.get('RunWhitelist'):
        requestDict['RunWhitelist'] = []
    if not requestDict.get('RunBlacklist'):
        requestDict['RunBlacklist'] = []
    if not requestDict.get('BlockWhitelist'):
        requestDict['BlockWhitelist'] = []
    if not requestDict.get('BlockBlacklist'):
        requestDict['BlockBlacklist'] = []
    
    compactRequest = {'InputDataset' : requestDict['InputDataset'],
                      'RunWhitelist' : requestDict['RunWhitelist'],
                      'RunBlacklist' : requestDict['RunBlacklist'],
                      'BlockWhitelist' : requestDict['BlockWhitelist'],
                      'BlockBlacklist' : requestDict['BlockBlacklist'],
                      'OutputDatasets' : []}
    response = requestor.get('/outputDatasetsByRequestName?requestName=%s' % requestName)
    if response[1] != 200:
        raise RuntimeError("Output dataset information was not available!")
    compactRequest['OutputDatasets'] = response[0]
    print "Done querying cmsweb... %s" % time.strftime('%H:%M:%S')
    return compactRequest

# 需要导入模块: from WMCore.Services.Requests import JSONRequests [as 别名]
# 或者: from WMCore.Services.Requests.JSONRequests import get [as 别名]
    def wmbsServiceSetup(self, argstring, kargs={}, returnType="text"):

        if returnType == "json":
            request = JSONRequests(self.server_url)
        else:
            request = Requests(self.server_url)
        results = request.get("/wmbs/%s/" % argstring, kargs)

        return results

# 需要导入模块: from WMCore.Services.Requests import JSONRequests [as 别名]
# 或者: from WMCore.Services.Requests.JSONRequests import get [as 别名]
def _verifyDBSCall(dbsURL, uri):
    try:
        # from WMCore.Services.DBS.DBS3Reader import DBS3Reader
        # DBS3Reader(dbsURL).dbs.serverinfo()
        from WMCore.Services.Requests import JSONRequests
        jsonSender = JSONRequests(dbsURL)
        result = jsonSender.get("/%s" % uri)
        if not result[1] == 200:
            raise WMSpecFactoryException("DBS is not connected: %s : %s" % (dbsURL, str(result)))
    except:
        raise WMSpecFactoryException("DBS is not responding: %s" % dbsURL)

    return result[0]

# 需要导入模块: from WMCore.Services.Requests import JSONRequests [as 别名]
# 或者: from WMCore.Services.Requests.JSONRequests import get [as 别名]
    def testPush(self):
        reg_info ={
                   "location": "https://globaldbs",
                   "admin": "[email protected]",
                   "type": "DBS",
                   "name": "Global DBS",
                   "timeout": 2
        }

        reg = Registration({'server': self.testInit.couchUrl, 'database': self.couch_db}, reg_info)

        json = BasicAuthJSONRequests(self.testInit.couchUrl)
        data = json.get('/%s/%s' % (self.couch_db, str(reg_info['location'].__hash__())))
        for k, v in reg_info.items():
            if k != 'timestamp':
                assert data[0][k] == v, \
                "Registration incomplete: %s should equal %s for key %s" % (data[0][k], v, k)

# 需要导入模块: from WMCore.Services.Requests import JSONRequests [as 别名]
# 或者: from WMCore.Services.Requests.JSONRequests import get [as 别名]
class RequestManagerTest(RESTBaseUnitTest):
    """
    Test RequestMgr Service client
    It will start RequestMgr RESTService
    Server DB is whatever env is set     
    
    This checks whether DS call makes without error and return the results.
    This test only test service call returns without error.
    The correctness of each function is tested in test/python/RequestManager_t/RequestMgr_t.py
    
    """
    def initialize(self):
        self.couchDBName = "reqmgr_t_0"
        self.config = RequestManagerConfig(
                'WMCore.HTTPFrontEnd.RequestManager.ReqMgrRESTModel')
        dbUrl = os.environ.get("DATABASE", None)
        self.config.setDBUrl(dbUrl)        
        self.config.setFormatter('WMCore.WebTools.RESTFormatter')
        self.config.setupRequestConfig()
        self.config.setupCouchDatabase(dbName = self.couchDBName)
        self.config.setPort(8899)
        self.schemaModules = ["WMCore.RequestManager.RequestDB"]
                
        
    def setUp(self):
        RESTBaseUnitTest.setUp(self)
        self.testInit.setupCouch("%s" % self.couchDBName, "GroupUser", "ConfigCache", "ReqMgr")
        self.testInit.setupCouch("%s_wmstats" % self.couchDBName, "WMStats")
        # logging stuff from TestInit is broken, setting myself
        l = logging.getLogger()
        l.setLevel(logging.DEBUG)
        self.params = {}
        self.params['endpoint'] = self.config.getServerUrl()
        self.reqService = RequestManagerDS(self.params)
        self.jsonSender = JSONRequests(self.config.getServerUrl())

        userName     = 'Taizong'
        groupName    = 'Li'
        teamName     = 'Tang'
        schema = utils.getAndSetupSchema(self,
                                         userName = userName,
                                         groupName = groupName,
                                         teamName = teamName)
        schema['ConfigCacheID'] = self.createConfig()
        schema['CouchDBName'] = self.couchDBName
        schema['CouchWorkloadDBName'] = self.couchDBName

        try:
            r = self.jsonSender.put('request', schema)
            try:
                self.requestName = r[0]['RequestName']
            except:
                self.requestName = r[0].values()[0]['RequestName']
        except Exception as ex:
            msg = traceback.format_exc()
            print("Exception during set up, reason: %s" % msg)
            raise ex

    def tearDown(self):
        self.config.deleteWorkloadCache()
        RESTBaseUnitTest.tearDown(self)
        self.testInit.tearDownCouch()

    def createConfig(self, bad = False):
        """
        _createConfig_

        Create a config of some sort that we can load out of ConfigCache
        """
        PSetTweak = {'process': {'outputModules_': ['ThisIsAName'],
                                 'ThisIsAName': {'dataset': {'dataTier': 'RECO',
                                                             'filterName': 'Filter'}}}}
        BadTweak  = {'process': {'outputModules_': ['ThisIsAName1', 'ThisIsAName2'],
                                 'ThisIsAName1': {'dataset': {'dataTier': 'RECO',
                                                             'filterName': 'Filter'}},
                                 'ThisIsAName2': {'dataset': {'dataTier': 'RECO',
                                                             'filterName': 'Filter'}}}}
        configCache = ConfigCache(os.environ["COUCHURL"], couchDBName = self.couchDBName)
        configCache.createUserGroup(groupname = "testGroup", username = 'testOps')
        if bad:
            configCache.setPSetTweaks(PSetTweak = BadTweak)
        else:
            configCache.setPSetTweaks(PSetTweak = PSetTweak)
        configCache.save()
        return configCache.getCouchID()

    @attr("integration")
    def testA_RequestManagerService(self):
        requestName = self.requestName
        
        request = self.reqService.getRequest(requestName)
        # minimal test : it's return type  and the some value inside
        self.assertEqual(type(request), dict)
        self.assertTrue(len(request) > 0)
        
        # Test putTeam
        self.reqService.putTeam("team_usa")
        self.assertTrue('team_usa' in self.jsonSender.get('team')[0])
        
        self.jsonSender.put('assignment/%s/%s' % ("team_usa", requestName))
#.........这里部分代码省略.........

# 需要导入模块: from WMCore.Services.Requests import JSONRequests [as 别名]
# 或者: from WMCore.Services.Requests.JSONRequests import get [as 别名]
class ReqMgrPriorityTest(RESTBaseUnitTest):
    """
    Basic test for setting the priority in ReqMgr Services
    
    """
    def setUp(self):
        """
        setUP global values
        Database setUp is done in base class
        
        """
        self.couchDBName = "reqmgr_t_0"
        RESTBaseUnitTest.setUp(self)
        self.testInit.setupCouch("%s" % self.couchDBName, "ConfigCache", "ReqMgr")
        self.testInit.setupCouch("%s_wmstats" % self.couchDBName, "WMStats")
        self.testInit.setupCouch("%s_acdc" % self.couchDBName, "ACDC")
        reqMgrHost = self.config.getServerUrl()
        self.jsonSender = JSONRequests(reqMgrHost)
        

    def initialize(self):
        self.config = RequestManagerConfig(
                'WMCore.HTTPFrontEnd.RequestManager.ReqMgrRESTModel')
        self.config.setFormatter('WMCore.WebTools.RESTFormatter')
        self.config.setupRequestConfig()
        self.config.setupCouchDatabase(dbName = self.couchDBName)
        self.config.setPort(8888)
        self.schemaModules = ["WMCore.RequestManager.RequestDB"]


    def tearDown(self):
        """
        _tearDown_

        Basic tear down of database
        
        """
        RESTBaseUnitTest.tearDown(self)
        self.testInit.tearDownCouch()
        

    def loadWorkload(self, requestName):
        """
        _loadWorkload_

        Load the workload from couch after we've saved it there.
        
        """
        workload = WMWorkloadHelper()
        url      = '%s/%s/%s/spec' % (os.environ['COUCHURL'], self.couchDBName,
                                      requestName)
        workload.load(url)
        return workload


    def changeStatusAndCheck(self, requestName, statusName):
        """
        _changeStatusAndCheck_

        Change the status of a request and make sure that
        the request actually did it.
        
        """
        self.jsonSender.put('request/%s?status=%s' % (requestName, statusName))
        result = self.jsonSender.get('request/%s' % requestName)
        self.assertEqual(result[0]['RequestStatus'], statusName)


    def testA_RequestPriority(self):
        """
        _priorityChanges_

        Do some fairly standard priority changes to the Request and
        see how things react
        
        """
        userName     = 'Taizong'
        groupName    = 'Li'
        teamName     = 'Tang'
        
        schema       = utils.getAndSetupSchema(self,
                                               userName = userName,
                                               groupName = groupName,
                                               teamName = teamName)
        result = self.jsonSender.put('request/testRequest', schema)
        self.assertEqual(result[1], 200)
        requestName = result[0]['RequestName']

        workload = self.loadWorkload(requestName = requestName)
        self.assertEqual(workload.priority(), 0)

        # Set priority == 5
        priority = 5
        self.jsonSender.put('request/%s?priority=%s' % (requestName, priority))
        request = self.jsonSender.get('request/%s' % requestName)[0]
        self.assertEqual(request['RequestPriority'], priority)
        workload = self.loadWorkload(requestName = requestName)
        self.assertEqual(workload.priority(), priority)

        # Set priority == 100
#.........这里部分代码省略.........

# 需要导入模块: from WMCore.Services.Requests import JSONRequests [as 别名]
# 或者: from WMCore.Services.Requests.JSONRequests import get [as 别名]
class TestReqMgr(RESTBaseUnitTest):
    """
    _TestReqMgr_

    Basic test for the ReqMgr services
    """

    def initialize(self):
        self.config = RequestManagerConfig(
                'WMCore.HTTPFrontEnd.RequestManager.ReqMgrRESTModel')
        dbUrl = os.environ.get("DATABASE", None)
        if dbUrl == None:
            raise RuntimeError, """The DATABASE environment is not set.  WARNING: setting this variable will cause the database to be deleted during cleanup! """
        self.config.setDBUrl(dbUrl)        
        self.config.setFormatter('WMCore.WebTools.RESTFormatter')
        self.config.setupRequestConfig()
        # mysql example
        #self.config.setDBUrl('mysql://[email protected]:3306/TestDB')
        #self.config.setDBSocket('/var/lib/mysql/mysql.sock')
        self.schemaModules = ["WMCore.RequestManager.RequestDB"]
        # Do this is you don't want to remake and delete the schema
        #self.schemaModules = []
        
    def setUp(self):
        """
        setUP global values
        """
        RESTBaseUnitTest.setUp(self)
        reqMgrHost = self.config.getServerUrl()
        self.requestSchema = getRequestSchema()
        print reqMgrHost
        self.jsonSender = JSONRequests(reqMgrHost)
        #self.requestTypes = ['ReReco', 'StoreResults', 'CmsGen', 'Reco']
        #self.requestTypes = ['ReReco', 'MonteCarlo']
        self.requestTypes = ['ReReco']

        if 'me' in self.jsonSender.get('user')[0]:
            self.jsonSender.delete('user/me')    
        self.assertFalse('me' in self.jsonSender.get('user')[0])
        self.assertEqual(self.jsonSender.put('user/[email protected]')[1], 200)
        self.assertTrue('me' in self.jsonSender.get('user')[0])

        if 'PeopleLikeMe' in self.jsonSender.get('group')[0]:
            self.jsonSender.delete('group/PeopleLikeMe')
        self.assertFalse('PeopleLikeMe' in self.jsonSender.get('group')[0])
        self.assertEqual(self.jsonSender.put('group/PeopleLikeMe')[1], 200)
        self.assertTrue( 'PeopleLikeMe' in self.jsonSender.get('group')[0])

        self.jsonSender.put('group/PeopleLikeMe/me')
        users = json.loads(self.jsonSender.get('group/PeopleLikeMe')[0])['users']
        self.assertTrue('me' in users)
        groups = json.loads(self.jsonSender.get('user/me')[0])['groups']
        self.assertTrue('PeopleLikeMe' in groups)
        groups2 = self.jsonSender.get('group?user=me')[0]
        self.assertTrue('PeopleLikeMe' in groups2)

        if 'White Sox' in self.jsonSender.get('team')[0]:
            self.jsonSender.delete(urllib.quote('team/White Sox'))
        self.assertFalse('White Sox' in self.jsonSender.get('team')[0])
        self.assertEqual(self.jsonSender.put(urllib.quote('team/White Sox'))[1], 200)
        self.assertTrue('White Sox' in self.jsonSender.get('team')[0])

        # some foreign key stuff to dealwith
        #self.assertFalse('CMSSW_X_Y_Z' in self.jsonSender.get('version')[0])
        self.assertTrue(self.jsonSender.put('version/CMSSW_3_5_8')[1] == 200)
        self.assertTrue('CMSSW_3_5_8' in self.jsonSender.get('version')[0])

    @attr("integration")
    def testReReco(self):
        schema = ReReco.getTestArguments()
        schema['RequestName'] = 'TestReReco'
        schema['RequestType'] = 'ReReco'
        schema['CmsPath'] = "/uscmst1/prod/sw/cms"
        self.doRequest(schema)

    def doRequest(self, schema):
        schema['CmsPath'] = "/uscmst1/prod/sw/cms"
        schema['Requestor'] = 'me'
        schema['Group'] = 'PeopleLikeMe'
        requestName = schema['RequestName']
        self.assertRaises(HTTPException, self.jsonSender.delete, 'request/%s' % requestName)
        self.assertEqual(self.jsonSender.put('request/%s' % requestName, schema)[1], 200)

        self.assertEqual(self.jsonSender.get('request/%s' % requestName)[0]['RequestName'], requestName)
        self.assertTrue(requestName in self.jsonSender.get('user/me')[0])

        self.jsonSender.put('request/%s?status=assignment-approved' % requestName)
        meJSON = self.jsonSender.get('user/me')[0]
        me = json.loads(meJSON)
        self.assertTrue(requestName in me['requests'])
        self.assertEqual(self.jsonSender.put('request/%s?priority=5' % requestName)[1], 200)
        self.assertEqual(self.jsonSender.post('user/me?priority=6')[1], 200)
        self.assertEqual(self.jsonSender.post('group/PeopleLikeMe?priority=7')[1], 200)

        # default priority of group and user of 1
        request = self.jsonSender.get('request/%s' % requestName)[0]
        self.assertEqual(request['ReqMgrRequestBasePriority'], 5)
        self.assertEqual(request['ReqMgrRequestorBasePriority'], 6)
        self.assertEqual(request['ReqMgrGroupBasePriority'], 7)
        self.assertEqual(request['RequestPriority'], 5+6+7)
#.........这里部分代码省略.........

# 需要导入模块: from WMCore.Services.Requests import JSONRequests [as 别名]
# 或者: from WMCore.Services.Requests.JSONRequests import get [as 别名]
class ReqMgrTest(RESTBaseUnitTest):
    """
    _ReqMgrTest_

    Basic test for the ReqMgr services.
    Setup is done off-screen in RESTBaseUnitTest - this makes
    things confusing
    """

    def setUp(self):
        """
        setUP global values
        Database setUp is done in base class
        """
        self.couchDBName = "reqmgr_t_0"
        RESTBaseUnitTest.setUp(self)
        self.testInit.setupCouch("%s" % self.couchDBName, "GroupUser", "ConfigCache")
        self.testInit.setupCouch("%s_wmstats" % self.couchDBName, "WMStats")

        reqMgrHost = self.config.getServerUrl()
        self.jsonSender = JSONRequests(reqMgrHost)
        return

    def initialize(self):
        self.config = RequestManagerConfig("WMCore.HTTPFrontEnd.RequestManager.ReqMgrRESTModel")
        self.config.setFormatter("WMCore.WebTools.RESTFormatter")
        self.config.setupRequestConfig()
        self.config.setupCouchDatabase(dbName=self.couchDBName)
        self.config.setPort(12888)
        self.schemaModules = ["WMCore.RequestManager.RequestDB"]
        return

    def tearDown(self):
        """
        tearDown 

        Tear down everything
        """
        RESTBaseUnitTest.tearDown(self)
        self.testInit.tearDownCouch()
        return

    def setupSchema(
        self,
        groupName="PeopleLikeMe",
        userName="me",
        teamName="White Sox",
        CMSSWVersion="CMSSW_3_5_8",
        scramArch="slc5_ia32_gcc434",
    ):
        """
        _setupSchema_

        Set up a test schema so that we can run a test request.
        Standardization!
        """

        self.jsonSender.put("user/%[email protected]" % userName)
        self.jsonSender.put("group/%s" % groupName)
        self.jsonSender.put("group/%s/%s" % (groupName, userName))
        self.jsonSender.put(urllib.quote("team/%s" % teamName))
        self.jsonSender.put("version/%s/%s" % (CMSSWVersion, scramArch))

        schema = ReReco.getTestArguments()
        schema["RequestName"] = "TestReReco"
        schema["RequestType"] = "ReReco"
        schema["CmsPath"] = "/uscmst1/prod/sw/cms"
        schema["Requestor"] = "%s" % userName
        schema["Group"] = "%s" % groupName
        schema["CustodialSite"] = "US_T1_FNAL"
        schema["TimePerEvent"] = "12"
        schema["Memory"] = 3000
        schema["SizePerEvent"] = 512

        return schema

    def createConfig(self, bad=False):
        """
        _createConfig_

        Create a config of some sort that we can load out of ConfigCache
        """

        PSetTweak = {
            "process": {
                "outputModules_": ["ThisIsAName"],
                "ThisIsAName": {"dataset": {"dataTier": "RECO", "filterName": "Filter"}},
            }
        }

        BadTweak = {
            "process": {
                "outputModules_": ["ThisIsAName1", "ThisIsAName2"],
                "ThisIsAName1": {"dataset": {"dataTier": "RECO", "filterName": "Filter"}},
                "ThisIsAName2": {"dataset": {"dataTier": "RECO", "filterName": "Filter"}},
            }
        }

        configCache = ConfigCache(os.environ["COUCHURL"], couchDBName=self.couchDBName)
        configCache.createUserGroup(groupname="testGroup", username="testOps")
#.........这里部分代码省略.........

# 需要导入模块: from WMCore.Services.Requests import JSONRequests [as 别名]
# 或者: from WMCore.Services.Requests.JSONRequests import get [as 别名]
class ReqMgrTest(RESTBaseUnitTest):
    """
    Basic test for the ReqMgr services.
    Setup is done off-screen in RESTBaseUnitTest - this makes
    things confusing
    
    """

    def setUp(self):
        """
        setUP global values
        Database setUp is done in base class
        """
        self.couchDBName = "reqmgr_t_0"
        RESTBaseUnitTest.setUp(self)
        self.testInit.setupCouch("%s" % self.couchDBName, "ConfigCache", "ReqMgr")
        self.testInit.setupCouch("%s_wmstats" % self.couchDBName,
                                 "WMStats")
        self.testInit.setupCouch("%s_acdc" % self.couchDBName,
                                 "ACDC", "GroupUser")
        reqMgrHost = self.config.getServerUrl()
        self.jsonSender = JSONRequests(reqMgrHost)

        self.params = {}
        self.params['endpoint'] = reqMgrHost
        self.reqService = RequestManager(self.params)

    def initialize(self):
        self.config = RequestManagerConfig(
                'WMCore.HTTPFrontEnd.RequestManager.ReqMgrRESTModel')
        self.config.setFormatter('WMCore.WebTools.RESTFormatter')
        self.config.setupRequestConfig()
        self.config.setupCouchDatabase(dbName = self.couchDBName)
        self.config.setPort(12888)
        self.schemaModules = ["WMCore.RequestManager.RequestDB"]


    def tearDown(self):
        """
        tearDown 

        Tear down everything
        
        """
        RESTBaseUnitTest.tearDown(self)
        self.testInit.tearDownCouch()


    def createConfig(self, bad = False):
        """
        _createConfig_

        Create a config of some sort that we can load out of ConfigCache
        """
        
        PSetTweak = {'process': {'outputModules_': ['ThisIsAName'],
                                 'ThisIsAName': {'dataset': {'dataTier': 'RECO',
                                                             'filterName': 'Filter'}}}}

        BadTweak  = {'process': {'outputModules_': ['ThisIsAName1', 'ThisIsAName2'],
                                 'ThisIsAName1': {'dataset': {'dataTier': 'RECO',
                                                             'filterName': 'Filter'}},
                                 'ThisIsAName2': {'dataset': {'dataTier': 'RECO',
                                                             'filterName': 'Filter'}}}}

        configCache = ConfigCache(os.environ["COUCHURL"], couchDBName = self.couchDBName)
        configCache.createUserGroup(groupname = "testGroup", username = 'testOps')
        if bad:
            configCache.setPSetTweaks(PSetTweak = BadTweak)
        else:
            configCache.setPSetTweaks(PSetTweak = PSetTweak)
        configCache.save()

        return configCache.getCouchID()
    

    @attr("integration")
    def testA_testBasicSetUp(self):
        """
        _testBasicSetUp_
        
        Moving the tests that were in the setUp category out of it,
        mostly because I want to make sure that they don't fail inside
        the setUp statement.
        
        """
        if 'me' in self.jsonSender.get('user')[0]:
            self.jsonSender.delete('user/me')    
        self.assertFalse('me' in self.jsonSender.get('user')[0])
        self.assertEqual(self.jsonSender.put('user/[email protected]')[1], 200)
        self.assertTrue('me' in self.jsonSender.get('user')[0])

        if 'PeopleLikeMe' in self.jsonSender.get('group')[0]:
            self.jsonSender.delete('group/PeopleLikeMe')
        self.assertFalse('PeopleLikeMe' in self.jsonSender.get('group')[0])
        self.assertEqual(self.jsonSender.put('group/PeopleLikeMe')[1], 200)
        self.assertTrue( 'PeopleLikeMe' in self.jsonSender.get('group')[0])

        self.jsonSender.put('group/PeopleLikeMe/me')
        users = self.jsonSender.get('group/PeopleLikeMe')[0]['users']
#.........这里部分代码省略.........

# 需要导入模块: from WMCore.Services.Requests import JSONRequests [as 别名]
# 或者: from WMCore.Services.Requests.JSONRequests import get [as 别名]
    def __call__(self, filesetToProcess):
        """
        The algorithm itself
        """
        global LOCK


        # Get configuration
        initObj = WMInit()
        initObj.setLogging()
        initObj.setDatabaseConnection(os.getenv("DATABASE"), \
            os.getenv('DIALECT'), os.getenv("DBSOCK"))

        myThread = threading.currentThread()

        daofactory = DAOFactory(package = "WMCore.WMBS" , \
              logger = myThread.logger, \
              dbinterface = myThread.dbi)

        locationNew = daofactory(classname = "Locations.New")
        getFileLoc = daofactory(classname = "Files.GetLocation")


        logging.debug("the T0Feeder is processing %s" % \
                 filesetToProcess.name)
        logging.debug("the fileset name %s" % \
         (filesetToProcess.name).split(":")[0])

        startRun = (filesetToProcess.name).split(":")[3]
        fileType = (filesetToProcess.name).split(":")[2]

        # url builder
        primaryDataset = ((filesetToProcess.name).split(":")[0]).split('/')[1]
        processedDataset = ((filesetToProcess.name).split(":")[0]).split('/')[2]
        dataTier = (((filesetToProcess.name\
            ).split(":")[0]).split('/')[3]).split('-')[0]

        # Fisrt call to T0 db for this fileset
        # Here add test for the closed fileset
        LASTIME = filesetToProcess.lastUpdate

        url = "/tier0/listfilesoverinterval/%s/%s/%s/%s/%s" % \
              (fileType, LASTIME, primaryDataset,processedDataset, dataTier)

        tries = 1
        while True:

            try:

                myRequester = JSONRequests(url = "vocms52.cern.ch:8889")
                requestResult = myRequester.get(\
             url+"/"+"?return_type=text/json%2Bdas")
                newFilesList = requestResult[0]["results"]

            except:

                logging.debug("T0Reader call error...")
                if tries == self.maxRetries:
                    return
                else:
                    tries += 1
                    continue

            logging.debug("T0ASTRun queries done ...")
            now = time.time()
            filesetToProcess.last_update = now
            LASTIME = int(newFilesList['end_time']) + 1

            break



        # process all files
        if len(newFilesList['files']):

            LOCK.acquire()

            try:
                locationNew.execute(siteName = "caf.cern.ch", seName = "caf.cern.ch")
            except Exception as e:
                logging.debug("Error when adding new location...")
                logging.debug(e)
                logging.debug( format_exc() )

            for files in newFilesList['files']:

                # Assume parents aren't asked
                newfile = File(str(files['lfn']), \
           size = files['file_size'], events = files['events'])


                try:
                    if newfile.exists() == False :
                        newfile.create()

                    else:
                        newfile.loadData()

                    #Add run test if already exist
                    for run in files['runs']:
#.........这里部分代码省略.........

# 需要导入模块: from WMCore.Services.Requests import JSONRequests [as 别名]
# 或者: from WMCore.Services.Requests.JSONRequests import get [as 别名]
            LOCK.release()

        else:

            logging.debug("nothing to do...")
            # For re-opned fileset or empty, try until the purge time
            if (int(now)/3600 - LASTIME/3600) > self.reopenTime:

                filesetToProcess.setLastUpdate(time.time())
                filesetToProcess.commit()


        if LASTIME:

            myRequester = JSONRequests(url = "vocms52.cern.ch:8889")
            requestResult = myRequester.get("/tier0/runs")

            for listRun in requestResult[0]:

                if int(startRun) <= int(listRun['run']):

                    if listRun['status'] =='CloseOutExport' or \
           listRun['status'] =='Complete' or listRun['status'] ==\
                          'CloseOutT1Skimming':

                        closeFileset = Fileset( name = (((\
      filesetToProcess.name).split(':')[0]).split('/')[0])+'/'+\
     (((filesetToProcess.name).split(':')[0]).split('/')[1]\
     )+'/'+(((filesetToProcess.name).split(':')[0]).split('/')\
     [2])+'/'+((((filesetToProcess.name).split(':')[0]).split\
     ('/')[3]).split('-')[0])+'-'+'Run'+str(listRun['run'])\

# 需要导入模块: from WMCore.Services.Requests import JSONRequests [as 别名]
# 或者: from WMCore.Services.Requests.JSONRequests import get [as 别名]
class ReqMgrPriorityTest(RESTBaseUnitTest):
    """
    Basic test for setting the priority in ReqMgr Services
    
    """

    def setUp(self):
        """
        setUP global values
        Database setUp is done in base class
        
        """
        self.couchDBName = "reqmgr_t_0"
        RESTBaseUnitTest.setUp(self)
        self.testInit.setupCouch("%s" % self.couchDBName, "ConfigCache", "ReqMgr")
        self.testInit.setupCouch("%s_wmstats" % self.couchDBName, "WMStats")
        self.testInit.setupCouch("%s_acdc" % self.couchDBName, "ACDC")
        reqMgrHost = self.config.getServerUrl()
        self.jsonSender = JSONRequests(reqMgrHost)

    def initialize(self):
        self.config = RequestManagerConfig("WMCore.HTTPFrontEnd.RequestManager.ReqMgrRESTModel")
        self.config.setFormatter("WMCore.WebTools.RESTFormatter")
        self.config.setupRequestConfig()
        self.config.setupCouchDatabase(dbName=self.couchDBName)
        self.config.setPort(8888)
        self.schemaModules = ["WMCore.RequestManager.RequestDB"]

    def tearDown(self):
        """
        _tearDown_

        Basic tear down of database
        
        """
        RESTBaseUnitTest.tearDown(self)
        self.testInit.tearDownCouch()

    def loadWorkload(self, requestName):
        """
        _loadWorkload_

        Load the workload from couch after we've saved it there.
        
        """
        workload = WMWorkloadHelper()
        url = "%s/%s/%s/spec" % (os.environ["COUCHURL"], self.couchDBName, requestName)
        workload.load(url)
        return workload

    def createConfig(self, bad=False):
        """
        _createConfig_

        Create a config of some sort that we can load out of ConfigCache
        
        """
        PSetTweak = {
            "process": {
                "outputModules_": ["ThisIsAName"],
                "ThisIsAName": {"dataset": {"dataTier": "RECO", "filterName": "Filter"}},
            }
        }
        BadTweak = {
            "process": {
                "outputModules_": ["ThisIsAName1", "ThisIsAName2"],
                "ThisIsAName1": {"dataset": {"dataTier": "RECO", "filterName": "Filter"}},
                "ThisIsAName2": {"dataset": {"dataTier": "RECO", "filterName": "Filter"}},
            }
        }
        configCache = ConfigCache(os.environ["COUCHURL"], couchDBName=self.couchDBName)
        configCache.createUserGroup(groupname="testGroup", username="testOps")
        if bad:
            configCache.setPSetTweaks(PSetTweak=BadTweak)
        else:
            configCache.setPSetTweaks(PSetTweak=PSetTweak)
        configCache.save()
        return configCache.getCouchID()

    def changeStatusAndCheck(self, requestName, statusName):
        """
        _changeStatusAndCheck_

        Change the status of a request and make sure that
        the request actually did it.
        
        """
        self.jsonSender.put("request/%s?status=%s" % (requestName, statusName))
        result = self.jsonSender.get("request/%s" % requestName)
        self.assertEqual(result[0]["RequestStatus"], statusName)

    def testA_RequestPriority(self):
        """
        _priorityChanges_

        Do some fairly standard priority changes to the Request and
        see how things react
        
        """
        userName = "Taizong"
#.........这里部分代码省略.........

# 需要导入模块: from WMCore.Services.Requests import JSONRequests [as 别名]
# 或者: from WMCore.Services.Requests.JSONRequests import get [as 别名]
class ReqMgrWorkloadTest(RESTBaseUnitTest):
    """
    Test that sets up and checks the validations of the various main WMSpec.StdSpecs
    This is mostly a simple set of tests which can be very repetitive.
    
    """

    def setUp(self):
        """
        setUP global values
        Database setUp is done in base class
        
        """
        self.couchDBName = "reqmgr_t_0"
        RESTBaseUnitTest.setUp(self)
        self.testInit.setupCouch("%s" % self.couchDBName, "ConfigCache", "ReqMgr")
        self.testInit.setupCouch("%s_wmstats" % self.couchDBName, "WMStats")
        reqMgrHost = self.config.getServerUrl()
        self.jsonSender = JSONRequests(reqMgrHost)


    def initialize(self):
        self.config = RequestManagerConfig(
                'WMCore.HTTPFrontEnd.RequestManager.ReqMgrRESTModel')
        self.config.setFormatter('WMCore.WebTools.RESTFormatter')
        self.config.setupRequestConfig()
        self.config.setupCouchDatabase(dbName = self.couchDBName)
        self.config.setPort(12888)
        self.schemaModules = ["WMCore.RequestManager.RequestDB"]


    def tearDown(self):
        """
        _tearDown_

        Basic tear down of database
        
        """
        RESTBaseUnitTest.tearDown(self)
        self.testInit.tearDownCouch()


    def createConfig(self, bad = False):
        """
        _createConfig_

        Create a config of some sort that we can load out of ConfigCache
        
        """
        PSetTweak = {'process': {'outputModules_': ['ThisIsAName'],
                                 'ThisIsAName': {'dataset': {'dataTier': 'RECO',
                                                             'filterName': 'Filter'}}}}
        BadTweak  = {'process': {'outputModules_': ['ThisIsAName1', 'ThisIsAName2'],
                                 'ThisIsAName1': {'dataset': {'dataTier': 'RECO',
                                                             'filterName': 'Filter'}},
                                 'ThisIsAName2': {'dataset': {'dataTier': 'RECO',
                                                             'filterName': 'Filter'}}}}
        configCache = ConfigCache(os.environ["COUCHURL"], couchDBName = self.couchDBName)
        configCache.createUserGroup(groupname = "testGroup", username = 'testOps')
        if bad:
            configCache.setPSetTweaks(PSetTweak = BadTweak)
        else:
            configCache.setPSetTweaks(PSetTweak = PSetTweak)
        configCache.save()
        return configCache.getCouchID()


    def loadWorkload(self, requestName):
        """
        _loadWorkload_

        Load the workload from couch after we've saved it there.
        
        """
        workload = WMWorkloadHelper()
        url      = '%s/%s/%s/spec' % (os.environ['COUCHURL'], self.couchDBName,
                                      requestName)
        workload.load(url)
        return workload
    

    @attr('integration')
    def testA_makeReRecoWorkload(self):
        """
        _makeReRecoWorkload_
 
        Check that you can make a basic ReReco workload
         
        """
        userName     = 'Taizong'
        groupName    = 'Li'
        teamName     = 'Tang'
        schema = utils.getAndSetupSchema(self,
                                         userName = userName,
                                         groupName = groupName,
                                         teamName = teamName)
        del schema['GlobalTag']
        configID = self.createConfig(bad = False)
        schema['ConfigCacheID'] = configID
        schema["CouchDBName"] = self.couchDBName
#.........这里部分代码省略.........

# 需要导入模块: from WMCore.Services.Requests import JSONRequests [as 别名]
# 或者: from WMCore.Services.Requests.JSONRequests import get [as 别名]
class ReqMgrTester(object):
    def __init__(self, reqMgrUrl):
        self.reqMgrUrl = reqMgrUrl
        self.restSender = JSONRequests(reqMgrUrl)
        d = dict(endpoint = self.reqMgrUrl)
        self.reqMgrService = RequestManager(d)
        

    def queryAllRequests(self):
        """
        Returns all requests stored at ReqMgr instance.
        
        """
        logging.info("Querying all requests at ReqMgr instance ...")
        r = self.reqMgrService.getRequestNames()
        print "Found %s requests:" % len(r)
        for req in r:
            print req
        

    def queryRequest(self, requestName):
        """
        Query a specific request according to the input argument.
        
        """
        urlQuery = "request/%s" % requestName
        logging.info("Querying request '%s'" % requestName)
        logging.info("Query: '%s':" % urlQuery)
        r = self.restSender.get(urlQuery)
        print str(r)
        
        
    def createRequests(self, numRequests):
        """
        Inject new numRequests into ReqMgr instance.
        (see ReqMgr_t testE how to create a request)
        
        """
        logging.info("Creating %s new requests ..." % numRequests)
        schema = ReReco.getTestArguments()
        schema['RequestName'] = 'TestReReco'
        schema['RequestType'] = 'ReReco'
        schema['CmsPath'] = "/uscmst1/prod/sw/cms"
        schema['Requestor'] = '%s' % "zmaxa"
        schema['Group'] = '%s' % "DATAOPS"
        schema['BlockWhitelist'] = ['/dataset/dataset/dataset#alpha']
        schema['BlockBlacklist'] = ['/dataset/dataset/dataset#beta']
        schema['Campaign'] = 'MyTestCampaign'        
        for i in range(numRequests):
            urlQuery = "request/testRequest"
            print "Query: '%s':" % urlQuery
            print "Schema (request): '%s'" % schema
            r = self.restSender.put(urlQuery, schema)
            # print "request creating response: ", r
            print "created: ", r[0]["RequestName"]  
                        
        
    def deleteRequest(self, requestNames):
        """
        Delete requests specified in the input, more request names
        are comma-separated.
        
        """
        logging.info("Deleting requests ...")
        for reqName in requestNames.split(','):
            reqName = reqName.strip()
            urlQuery = "request/%s" % reqName
            logging.info("Deleting request (request_name): '%s'" % reqName)
            logging.info("Query: '%s':" % urlQuery)
            r = self.restSender.delete(urlQuery)
            
            
    def requestChangeStates(self, reqName):
        """
        Route the request (spec. by the request name) in the input
        through a series of possible request states.
        
        """
        logging.info("Changing state of request %s ..." % reqName)
        def changeState(requestName, urlQuery):
            logging.info("Query: '%s' ..." % urlQuery)
            r = self.restSender.put(urlQuery)
            r = self.restSender.get("request/%s" % requestName)
            #assert r[0]["RequestStatus"] == statusName
            logging.info("Querying modified request, new state: %s" % r[0]["RequestStatus"])

        # once a request is created, it's in 'new' state
        # states transition has to be an allowed one as defined here:
        # WMCore/RequestManager/RequestDB/Settings/RequestStatus.py
        statesQueries = ["request/%s?status=%s" % (reqName, "testing-approved"),
                         "request/%s?status=%s" % (reqName, "testing"),
                         "request/%s?status=%s" % (reqName, "tested"),
                         "request/%s?status=%s" % (reqName, "assignment-approved"),
                         # create an assignment now
                         # need quote because of space in the team name
                         # (previous name - White Sox)
                         urllib.quote("assignment/%s/%s" % (TEAM_NAME, reqName))]
        for query in statesQueries:
            changeState(reqName, query)
            
#.........这里部分代码省略.........