Python SiteDB.SiteDBJSON类代码示例

    def execute(self, *args, **kwargs):

        totalevents = kwargs['task']['tm_totalunits']
        firstEvent = 1
        lastEvent = totalevents
        firstLumi = 1
        lastLumi = 10

        # Set a default of 100 events per lumi.  This is set as a task
        # property, as the splitting considers it independently of the file
        # information provided by the fake dataset.
        if not kwargs['task']['tm_events_per_lumi']:
            kwargs['task']['tm_events_per_lumi'] = 100

        #MC comes with only one MCFakeFile
        singleMCFileset = Fileset(name = "MCFakeFileSet")
        newFile = File("MCFakeFile", size = 1000, events = totalevents)
        sbj = SiteDBJSON({"key":self.config.TaskWorker.cmskey,
                          "cert":self.config.TaskWorker.cmscert})
        newFile.setLocation(sbj.getAllCMSNames())
        newFile.addRun(Run(1, *range(firstLumi, lastLumi + 1)))
        newFile["block"] = 'MCFakeBlock'
        newFile["first_event"] = firstEvent
        newFile["last_event"] = lastEvent
        singleMCFileset.addFile(newFile)

        return Result(task=kwargs['task'], result=singleMCFileset)

    def __init__(self, config, noSiteDB=False):
        """
        _init_

        Note, noSiteDB added for TESTING PURPOSED ONLY!
        """
        WebAPI.__init__(self, config)
        ReqMgrAuth.assign_roles = config.security_roles
        # Take a guess
        self.templatedir = config.templates
        self.couchUrl = config.couchUrl
        self.configDBName = config.configDBName
        self.workloadDBName = config.workloadDBName
        self.configDBName = config.configDBName
        self.wmstatWriteURL = "%s/%s" % (self.couchUrl.rstrip("/"), config.wmstatDBName)
        if not noSiteDB:
            try:
                # Download a list of all the sites from SiteDB, uses v2 API.
                sitedb = SiteDBJSON()
                self.sites = sitedb.getAllCMSNames()
                self.sites.sort()
            except Exception, ex:
                msg = "ERROR: Could not retrieve sites from SiteDB, reason: %s" % ex
                cherrypy.log(msg)
                raise

 def getUsernameFromSiteDB(self):
     """
     Retrieve the user's username as it appears in SiteDB.
     """
     proxy = self.proxy()
     userdn = proxy.getSubjectFromCert(self.certLocation)
     sitedb = SiteDBJSON({"key": proxy.getProxyFilename(), "cert": proxy.getProxyFilename()})
     username = sitedb.dnUserName(userdn)
     return username

 def getUsernameFromSiteDB(self):
     """
     Return a the client hypernews name
     """
     proxy = self.proxy()
     userdn = proxy.getSubjectFromCert(self.certLocation)
     sitedb = SiteDBJSON({"key": proxy.getProxyFilename(), "cert": proxy.getProxyFilename()})
     username = sitedb.dnUserName(userdn)
     return username

def sites():
    "Return known CMS site list from SiteDB"
    try:
        # Download a list of all the sites from SiteDB, uses v2 API.
        sitedb = SiteDBJSON()
        sites = sorted(sitedb.getAllCMSNames())
    except Exception as exc:
        msg = "ERROR: Could not retrieve sites from SiteDB, reason: %s" % str(exc)
        raise Exception(msg)
    return sites

def pnns():
    """
    Returns all PhEDEx node names, excluding Buffer endpoints
    """
    try:
        sitedb = SiteDBJSON()
        pnns = sorted(sitedb.getAllPhEDExNodeNames(excludeBuffer=True))
    except Exception as exc:
        msg = "ERROR: Could not retrieve PNNs from SiteDB, reason: %s" % str(exc)
        raise Exception(msg)
    return pnns

 def gethnName_urlenc(self,dn):
     from WMCore.Services.SiteDB.SiteDB import SiteDBJSON
     hnUserName = None
     userdn = dn
     mySiteDB = SiteDBJSON()
     status = 0 
     try:
         hnUserName = mySiteDB.dnUserName(dn=userdn)
     except:
         status = 1 
     return status,hnUserName

    def __init__(self, taskid):
        super(Monitor, self).__init__(taskid)

        p = subprocess.Popen(["voms-proxy-info", "-identity"],
                stdout=subprocess.PIPE,
                stderr=subprocess.PIPE)
        id, err = p.communicate()
        id = id.strip()
        db = SiteDBJSON({'cacheduration': 24})

        self.__username = db.dnUserName(dn=id)
        self.__fullname = id.rsplit('/CN=', 1)[1]

def cmsSiteNames():
    """Get all cms sites"""
    global __cmsSiteNames
    if __cmsSiteNames:
        return __cmsSiteNames
    global __sitedb
    if not __sitedb:
        from WMCore.Services.SiteDB.SiteDB import SiteDBJSON as SiteDB
        __sitedb = SiteDB()
    try:
        __cmsSiteNames = __sitedb.getAllCMSNames()
    except:
        pass
    return __cmsSiteNames

def getDNFromUserName(username, log, ckey = None, cert = None):
    """
    Parse site string to know the fts server to use
    """
    dn = ''
    site_db = SiteDBJSON(config={'key': ckey, 'cert': cert})
    try:
       dn = site_db.userNameDn(username)
    except IndexError:
       log.error("user does not exist")
       return dn
    except RuntimeError:
       log.error("SiteDB URL cannot be accessed")
       return dn
    return dn

def sitesFromStorageEelements(ses):
    """Return Sites given Storage Elements"""
    global __sitedb
    if not __sitedb:
        from WMCore.Services.SiteDB.SiteDB import SiteDBJSON as SiteDB
        __sitedb = SiteDB()
    result = set()
    for se in ses:
        try:
            sites = __sitedb.seToCMSName(se)
        except:
            print "Unable to get site name for %s" % se
        else:
            result.update(sites)
    return list(result)

 def setUp(self):
     """
     Setup for unit tests
     """
     super(SiteDBTest, self).setUp()
     EmulatorHelper.setEmulators(phedex=False, dbs=False, siteDB=False, requestMgr=True)
     self.mySiteDB = SiteDBJSON()

    def __init__(self, config):
        """
        ___init___

        Initialise class members
        """
        BaseWorkerThread.__init__(self)
        self.phedex = PhEDEx({"endpoint": config.PhEDExInjector.phedexurl}, "json")
        self.siteDB = SiteDBJSON()
        self.dbsUrl = config.DBSInterface.globalDBSUrl
        self.group = getattr(config.PhEDExInjector, "group", "DataOps")
        self.safeMode = getattr(config.PhEDExInjector, "safeOperationMode", False)

        # Subscribed state in the DBSBuffer table for datasets
        self.terminalSubscriptionState = 1
        if self.safeMode:
            self.terminalSubscriptionState = 2

        # We will map node names to CMS names, that what the spec will have.
        # If a CMS name is associated to many PhEDEx node then choose the MSS option
        self.cmsToPhedexMap = {}

        # initialize the alert framework (if available - config.Alert present)
        #    self.sendAlert will be then be available    
        self.initAlerts(compName = "PhEDExInjector")

    def execute(self, *args, **kwargs):
        self.logger.info("Data discovery and splitting for %s using user-provided files" % kwargs['task']['tm_taskname'])

        if 'tm_user_files' in kwargs['task'] and kwargs['task']['tm_user_files']:
            userfiles = kwargs['task']['tm_user_files']
        else: ## For backward compatibility only.
            userfiles = kwargs['task']['tm_arguments'].get('userfiles')
        splitting = kwargs['task']['tm_split_algo']
        total_units = kwargs['task']['tm_totalunits']
        if not userfiles or splitting != 'FileBased':
            if not userfiles:
                msg = "No files specified to process for task %s." % kwargs['task']['tm_taskname']
            if splitting != 'FileBased':
                msg = "Data.splitting must be set to 'FileBased' when using a custom set of files."
            self.logger.error("Setting %s as failed: %s" % (kwargs['task']['tm_taskname'], msg))
            configreq = {'workflow': kwargs['task']['tm_taskname'],
                         'status': "FAILED",
                         'subresource': 'failure',
                         'failure': b64encode(msg)}
            self.server.post(self.resturi, data = urllib.urlencode(configreq))
            raise StopHandler(msg)

        if hasattr(self.config.Sites, 'available'):
            locations = self.config.Sites.available
        else:
            sbj = SiteDBJSON({"key":self.config.TaskWorker.cmskey,
                              "cert":self.config.TaskWorker.cmscert})
            locations = sbj.getAllCMSNames()

        userFileset = Fileset(name = kwargs['task']['tm_taskname'])
        self.logger.info("There are %d files specified by the user." % len(userfiles))
        if total_units > 0:
            self.logger.info("Will run over the first %d files." % total_units)
        file_counter = 0
        for userfile, idx in zip(userfiles, range(len(userfiles))):
            newFile = File(userfile, size = 1000, events = 1)
            newFile.setLocation(locations)
            newFile.addRun(Run(1, idx))
            newFile["block"] = 'UserFilesFakeBlock'
            newFile["first_event"] = 1
            newFile["last_event"] = 2
            userFileset.addFile(newFile)
            file_counter += 1
            if total_units > 0 and file_counter >= total_units:
                break

        return Result(task = kwargs['task'], result = userFileset)

def phedexIt():
    x = PhEDEx(responseType = "json")
    phedexNodes = x.getNodeMap()['phedex']['node']
    phedexMap = {}
    sePhedexMap = {}
    knownPhedexNodes = set()
    for node in phedexNodes:
        phedexMap[node['name']] = node['kind']
        #print '%s -> %s, %s' % (node['name'], node['kind'], node['se'])
        if node['se'] not in sePhedexMap:
            sePhedexMap[node['se']] = set()
        sePhedexMap[node['se']].add(node['name'])
        knownPhedexNodes.add(node['name'])
    y = SiteDBJSON()
    seNames = y.getAllSENames()
    cmsNamesMap = {}
    for se in seNames:
        cmsNames = y.seToCMSName(se)
        cmsNamesMap[se] = cmsNames
    seToNodeMap = {}
    for se in cmsNamesMap:
        candidates = set()
        for cmsName in cmsNamesMap[se]:
            phedexNodes = y.cmsNametoPhEDExNode(cmsName)
            candidates.update(set(phedexNodes))
        validCandidates = set()
        for candidate in candidates:
            if candidate in knownPhedexNodes:
                validCandidates.add(candidate)
        seToNodeMap[se] = validCandidates
        #print '%s to %s' % (se, candidates)
    for se in sePhedexMap:
        if se not in seToNodeMap:
            print "SE: %s is not in new mapping for sites %s" % (se, list(sePhedexMap[se]))
    for se in seToNodeMap:
        if se not in sePhedexMap:
            print "SE: %s is not in old mapping for sites %s" % (se, list(seToNodeMap[se]))
            continue
    for se in set(seToNodeMap.keys()).intersection(set(sePhedexMap.keys())):
        diff = sePhedexMap[se] - seToNodeMap[se]
        if diff:
            print "%s are in old mapping but not in new for %s" %(str(list(diff)), se)
        diff = seToNodeMap[se] - sePhedexMap[se]
        if diff:
            print "%s are in new mapping but not in old for %s" %(str(list(diff)), se)