Python SiteDBJSON.getAllCMSNames方法代码示例

# 需要导入模块: from WMCore.Services.SiteDB.SiteDB import SiteDBJSON [as 别名]
# 或者: from WMCore.Services.SiteDB.SiteDB.SiteDBJSON import getAllCMSNames [as 别名]
    def execute(self, *args, **kwargs):

        totalevents = kwargs['task']['tm_totalunits']
        firstEvent = 1
        lastEvent = totalevents
        firstLumi = 1
        lastLumi = 10

        # Set a default of 100 events per lumi.  This is set as a task
        # property, as the splitting considers it independently of the file
        # information provided by the fake dataset.
        if not kwargs['task']['tm_events_per_lumi']:
            kwargs['task']['tm_events_per_lumi'] = 100

        #MC comes with only one MCFakeFile
        singleMCFileset = Fileset(name = "MCFakeFileSet")
        newFile = File("MCFakeFile", size = 1000, events = totalevents)
        sbj = SiteDBJSON({"key":self.config.TaskWorker.cmskey,
                          "cert":self.config.TaskWorker.cmscert})
        newFile.setLocation(sbj.getAllCMSNames())
        newFile.addRun(Run(1, *range(firstLumi, lastLumi + 1)))
        newFile["block"] = 'MCFakeBlock'
        newFile["first_event"] = firstEvent
        newFile["last_event"] = lastEvent
        singleMCFileset.addFile(newFile)

        return Result(task=kwargs['task'], result=singleMCFileset)

# 需要导入模块: from WMCore.Services.SiteDB.SiteDB import SiteDBJSON [as 别名]
# 或者: from WMCore.Services.SiteDB.SiteDB.SiteDBJSON import getAllCMSNames [as 别名]
    def __init__(self, config, noSiteDB=False):
        """
        _init_

        Note, noSiteDB added for TESTING PURPOSED ONLY!
        """
        WebAPI.__init__(self, config)
        ReqMgrAuth.assign_roles = config.security_roles
        # Take a guess
        self.templatedir = config.templates
        self.couchUrl = config.couchUrl
        self.configDBName = config.configDBName
        self.workloadDBName = config.workloadDBName
        self.configDBName = config.configDBName
        self.wmstatWriteURL = "%s/%s" % (self.couchUrl.rstrip("/"), config.wmstatDBName)
        if not noSiteDB:
            try:
                # Download a list of all the sites from SiteDB, uses v2 API.
                sitedb = SiteDBJSON()
                self.sites = sitedb.getAllCMSNames()
                self.sites.sort()
            except Exception, ex:
                msg = "ERROR: Could not retrieve sites from SiteDB, reason: %s" % ex
                cherrypy.log(msg)
                raise

# 需要导入模块: from WMCore.Services.SiteDB.SiteDB import SiteDBJSON [as 别名]
# 或者: from WMCore.Services.SiteDB.SiteDB.SiteDBJSON import getAllCMSNames [as 别名]
def sites():
    "Return known CMS site list from SiteDB"
    try:
        # Download a list of all the sites from SiteDB, uses v2 API.
        sitedb = SiteDBJSON()
        sites = sorted(sitedb.getAllCMSNames())
    except Exception as exc:
        msg = "ERROR: Could not retrieve sites from SiteDB, reason: %s" % str(exc)
        raise Exception(msg)
    return sites

# 需要导入模块: from WMCore.Services.SiteDB.SiteDB import SiteDBJSON [as 别名]
# 或者: from WMCore.Services.SiteDB.SiteDB.SiteDBJSON import getAllCMSNames [as 别名]
class MakeFakeFileSet(TaskAction):
    """This is needed to make WMCore.JobSplitting lib working...
       do not like very much. Given that all is fake here I am
       quite sure we only need total number of events said that I
       set all the other parmas to dummy values. We may want to set
       them in the future"""

    def __init__(self, *args, **kwargs):
        TaskAction.__init__(self, *args, **kwargs)
        self.sbj = SiteDBJSON({"key":self.config.TaskWorker.cmskey,
                          "cert":self.config.TaskWorker.cmscert})


    def getListOfSites(self):
        """ Get the list of sites to use for PrivateMC workflows.
            For the moment we are filtering out T1_ since they are precious resources
            and don't want to overtake production (WMAgent) jobs there. In the
            future we would like to take this list from the SSB.
        """
        sites = self.sbj.getAllCMSNames()
        filteredSites = [site for site in sites if not site.startswith("T1_")]

        return filteredSites


    #even though args is not used we call "handler.actionWork(args, kwargs)" in Handler
    def execute(self, *args, **kwargs): #pylint: disable=unused-argument

        totalevents = kwargs['task']['tm_totalunits']
        firstEvent = 1
        lastEvent = totalevents
        firstLumi = 1
        lastLumi = 10

        # Set a default of 100 events per lumi.  This is set as a task
        # property, as the splitting considers it independently of the file
        # information provided by the fake dataset.
        if not kwargs['task']['tm_events_per_lumi']:
            kwargs['task']['tm_events_per_lumi'] = 100

        #MC comes with only one MCFakeFile
        singleMCFileset = Fileset(name = "MCFakeFileSet")
        newFile = File("MCFakeFile", size = 1000, events = totalevents)
        newFile.setLocation(self.getListOfSites())
        newFile.addRun(Run(1, *range(firstLumi, lastLumi + 1)))
        newFile["block"] = 'MCFakeBlock'
        newFile["first_event"] = firstEvent
        newFile["last_event"] = lastEvent
        singleMCFileset.addFile(newFile)

        return Result(task=kwargs['task'], result=singleMCFileset)

# 需要导入模块: from WMCore.Services.SiteDB.SiteDB import SiteDBJSON [as 别名]
# 或者: from WMCore.Services.SiteDB.SiteDB.SiteDBJSON import getAllCMSNames [as 别名]
def cmsSiteNames():
    """Get all cms sites"""
    global __cmsSiteNames
    if __cmsSiteNames:
        return __cmsSiteNames
    global __sitedb
    if not __sitedb:
        from WMCore.Services.SiteDB.SiteDB import SiteDBJSON as SiteDB
        __sitedb = SiteDB()
    try:
        __cmsSiteNames = __sitedb.getAllCMSNames()
    except:
        pass
    return __cmsSiteNames

# 需要导入模块: from WMCore.Services.SiteDB.SiteDB import SiteDBJSON [as 别名]
# 或者: from WMCore.Services.SiteDB.SiteDB.SiteDBJSON import getAllCMSNames [as 别名]
    def execute(self, *args, **kwargs):
        self.logger.info("Data discovery and splitting for %s using user-provided files" % kwargs['task']['tm_taskname'])

        if 'tm_user_files' in kwargs['task'] and kwargs['task']['tm_user_files']:
            userfiles = kwargs['task']['tm_user_files']
        else: ## For backward compatibility only.
            userfiles = kwargs['task']['tm_arguments'].get('userfiles')
        splitting = kwargs['task']['tm_split_algo']
        total_units = kwargs['task']['tm_totalunits']
        if not userfiles or splitting != 'FileBased':
            if not userfiles:
                msg = "No files specified to process for task %s." % kwargs['task']['tm_taskname']
            if splitting != 'FileBased':
                msg = "Data.splitting must be set to 'FileBased' when using a custom set of files."
            self.logger.error("Setting %s as failed: %s" % (kwargs['task']['tm_taskname'], msg))
            configreq = {'workflow': kwargs['task']['tm_taskname'],
                         'status': "FAILED",
                         'subresource': 'failure',
                         'failure': b64encode(msg)}
            self.server.post(self.resturi, data = urllib.urlencode(configreq))
            raise StopHandler(msg)

        if hasattr(self.config.Sites, 'available'):
            locations = self.config.Sites.available
        else:
            sbj = SiteDBJSON({"key":self.config.TaskWorker.cmskey,
                              "cert":self.config.TaskWorker.cmscert})
            locations = sbj.getAllCMSNames()

        userFileset = Fileset(name = kwargs['task']['tm_taskname'])
        self.logger.info("There are %d files specified by the user." % len(userfiles))
        if total_units > 0:
            self.logger.info("Will run over the first %d files." % total_units)
        file_counter = 0
        for userfile, idx in zip(userfiles, range(len(userfiles))):
            newFile = File(userfile, size = 1000, events = 1)
            newFile.setLocation(locations)
            newFile.addRun(Run(1, idx))
            newFile["block"] = 'UserFilesFakeBlock'
            newFile["first_event"] = 1
            newFile["last_event"] = 2
            userFileset.addFile(newFile)
            file_counter += 1
            if total_units > 0 and file_counter >= total_units:
                break

        return Result(task = kwargs['task'], result = userFileset)

# 需要导入模块: from WMCore.Services.SiteDB.SiteDB import SiteDBJSON [as 别名]
# 或者: from WMCore.Services.SiteDB.SiteDB.SiteDBJSON import getAllCMSNames [as 别名]
    def insertAllSEs(self, siteName, pendingSlots = 0, runningSlots = 0,
                     ceName = None, plugin = None,
                     taskList = []):
        """
        _insertAllSEs_

        Insert all SEs into WMBS ResourceControl
        This uses the Services.SiteDB to insert all SEs under a common
        CE.  It is meant to be used with WMS submission.

        Sites will be named siteName_SEName

        It expects a taskList of the following form:

        [{'taskType': taskType, 'priority': priority, 'maxSlots': maxSlots}]

        for each entry in the taskList, a threshold is inserted into the database
        for EVERY SE
        """

        from WMCore.Services.SiteDB.SiteDB import SiteDBJSON
        siteDB = SiteDBJSON()

        cmsNames = siteDB.getAllCMSNames()
        for cmsName in cmsNames:
            seNames = siteDB.cmsNametoSE(cmsName)
            for SE in seNames:
                sName = '%s_%s' % (siteName, SE)
                self.insertSite(siteName = sName, pendingSlots = pendingSlots,
                                seName = SE, runningSlots = runningSlots,
                                ceName = ceName, cmsName = cmsName, plugin = plugin)
                for task in taskList:
                    if not task.has_key('maxSlots') or not task.has_key('taskType') \
                           or not task.has_key('priority'):
                        msg =  "Incomplete task in taskList for ResourceControl.insertAllSEs\n"
                        msg += task
                        raise ResourceControlException(msg)
                    self.insertThreshold(siteName = sName, taskType = task['taskType'],
                                         maxSlots = task['maxSlots'], priority = task['priority'])


        return

# 需要导入模块: from WMCore.Services.SiteDB.SiteDB import SiteDBJSON [as 别名]
# 或者: from WMCore.Services.SiteDB.SiteDB.SiteDBJSON import getAllCMSNames [as 别名]
def getSiteInfo(config):
    sitedb = SiteDBJSON()
    sites = sitedb.getAllCMSNames()
    sites.sort()
    wildcardKeys = getattr(config, 'wildcardKeys', {'T1*': 'T1_*',
                                                    'T2*': 'T2_*',
                                                    'T3*': 'T3_*'})
    wildcardSites = {}

    for k in wildcardKeys.keys():
        reValue = wildcardKeys.get(k)
        found   = False
        for s in sites:
            if re.search(reValue, s):
                found = True
                if not k in wildcardSites.keys():
                    wildcardSites[k] = []
                wildcardSites[k].append(s)
        if found:
            sites.append(k)
    return sites

# 需要导入模块: from WMCore.Services.SiteDB.SiteDB import SiteDBJSON [as 别名]
# 或者: from WMCore.Services.SiteDB.SiteDB.SiteDBJSON import getAllCMSNames [as 别名]
    def execute(self, *args, **kwargs):
        self.logger.info("Data discovery and splitting for %s using user-provided files" % kwargs['task']['tm_taskname'])

        userfiles = kwargs['task']['tm_user_files']
        splitting = kwargs['task']['tm_split_algo']
        total_units = kwargs['task']['tm_totalunits']
        if not userfiles or splitting != 'FileBased':
            if not userfiles:
                msg = "No files specified to process for task %s." % kwargs['task']['tm_taskname']
            if splitting != 'FileBased':
                msg = "Data.splitting must be set to 'FileBased' when using a custom set of files."
            raise TaskWorkerException(msg)

        if hasattr(self.config.Sites, 'available'):
            locations = self.config.Sites.available
        else:
            sbj = SiteDBJSON({"key":self.config.TaskWorker.cmskey,
                              "cert":self.config.TaskWorker.cmscert})
            locations = sbj.getAllCMSNames()

        userFileset = Fileset(name = kwargs['task']['tm_taskname'])
        self.logger.info("There are %d files specified by the user." % len(userfiles))
        if total_units > 0:
            self.logger.info("Will run over the first %d files." % total_units)
        file_counter = 0
        for userfile, idx in zip(userfiles, range(len(userfiles))):
            newFile = File(userfile, size = 1000, events = 1)
            newFile.setLocation(locations)
            newFile.addRun(Run(1, idx))
            newFile["block"] = 'UserFilesFakeBlock'
            newFile["first_event"] = 1
            newFile["last_event"] = 2
            userFileset.addFile(newFile)
            file_counter += 1
            if total_units > 0 and file_counter >= total_units:
                break

        return Result(task = kwargs['task'], result = userFileset)

# 需要导入模块: from WMCore.Services.SiteDB.SiteDB import SiteDBJSON [as 别名]
# 或者: from WMCore.Services.SiteDB.SiteDB.SiteDBJSON import getAllCMSNames [as 别名]
    def __init__(self, config, noSiteDB=False):
        """
        _init_

        Note, noSiteDB added for TESTING PURPOSED ONLY!
        """
        WebAPI.__init__(self, config)
        ReqMgrAuth.assign_roles = config.security_roles
        # Take a guess
        self.templatedir = config.templates
        self.couchUrl = config.couchUrl
        self.configDBName = config.configDBName
        self.workloadDBName = config.workloadDBName
        self.configDBName = config.configDBName
        self.wmstatWriteURL = "%s/%s" % (self.couchUrl.rstrip("/"), config.wmstatDBName)
        if not noSiteDB:
            try:
                # Download a list of all the sites from SiteDB, uses v2 API.
                sitedb = SiteDBJSON()
                self.sites = sitedb.getAllCMSNames()
                self.sites.sort()
                self.phedexNodes = sitedb.getAllPhEDExNodeNames(excludeBuffer=True)
                self.phedexNodes.sort()
            except Exception as ex:
                msg = "ERROR: Could not retrieve sites from SiteDB, reason: %s" % ex
                cherrypy.log(msg)
                raise
        else:
            self.sites = []

        # store result lfn base with all Physics group
        storeResultLFNBase = [
            "/store/results/analysisops",
            "/store/results/b_physics",
            "/store/results/b_tagging",
            "/store/results/b2g",
            "/store/results/e_gamma_ecal",
            "/store/results/ewk",
            "/store/results/exotica",
            "/store/results/forward",
            "/store/results/heavy_ions",
            "/store/results/higgs",
            "/store/results/jets_met_hcal",
            "/store/results/muon",
            "/store/results/qcd",
            "/store/results/susy",
            "/store/results/tau_pflow",
            "/store/results/top",
            "/store/results/tracker_dpg",
            "/store/results/tracker_pog",
            "/store/results/trigger",
        ]
        # yet 0.9.40 had also another self.mergedLFNBases which was differentiating
        # list of mergedLFNBases based on type of request, removed and all bases
        # will be displayed regardless of the request type (discussion with Edgar)
        self.allMergedLFNBases = [
            "/store/backfill/1",
            "/store/backfill/2",
            "/store/data",
            "/store/mc",
            "/store/generator",
            "/store/relval",
            "/store/hidata",
            "/store/himc",
        ]

        self.allMergedLFNBases.extend(storeResultLFNBase)

        self.allUnmergedLFNBases = ["/store/unmerged", "/store/temp"]

        self.yuiroot = config.yuiroot
        cherrypy.engine.subscribe("start_thread", self.initThread)

        self.wildcardKeys = getattr(config, "wildcardKeys", {"T1*": "T1_*", "T2*": "T2_*", "T3*": "T3_*"})
        self.wildcardSites = {}
        Utilities.addSiteWildcards(self.wildcardKeys, self.sites, self.wildcardSites)

# 需要导入模块: from WMCore.Services.SiteDB.SiteDB import SiteDBJSON [as 别名]
# 或者: from WMCore.Services.SiteDB.SiteDB.SiteDBJSON import getAllCMSNames [as 别名]
#! /bin/env python

import ConfigParser
import sys

ConfigFile = '/etc/ciconnect/resources.ini'

# Get CMS Sites from SiteDB
# This requires the user to setup CMSSW and CRAB3 or WMCore
# before being able to use the script

from WMCore.Services.SiteDB.SiteDB import SiteDBJSON
db = SiteDBJSON({'cacheduration': 24})
cmsnames = db.getAllCMSNames()
if len(cmsnames) > 10:
    sitelist = ",".join(cmsnames)
else:
    sys.exit("[Error]: Couldn't get a reasonable list of sites")
# print sitelist

cfg = ConfigParser.RawConfigParser()
cfg.read(ConfigFile)
if cfg.has_section('resources'):
    cfg.set('resources', 'ListOfSites', sitelist)

resfile = open(ConfigFile,'w')
cfg.write(resfile)

print "All Done."