Python ConfigCache.loadByID方法代码示例

# 需要导入模块: from WMCore.Cache.WMConfigCache import ConfigCache [as 别名]
# 或者: from WMCore.Cache.WMConfigCache.ConfigCache import loadByID [as 别名]
    def __call__(self, wmTask):
        """
        Trip through steps, find CMSSW steps, pull in config files,
        PSet Tweaks etc

        """
        for t in wmTask.steps().nodeIterator():
            t = WMStep.WMStepHelper(t)
            stepPath = "%s/%s" % (self.workingDirectory(), t.name())

            # the CMSSW has a special case with its ConfigCache argument
            if not t.stepType() in ("CMSSW", "MulticoreCMSSW"): continue
            if getattr(t.data.application.configuration, 'configCacheUrl', None) != None:
                # main config file
                fileTarget = "%s/%s" % (
                    stepPath,
                    t.data.application.command.configuration)
                #urllib.urlretrieve(
                #    t.data.application.configuration.retrieveConfigUrl,
                #    fileTarget)
                # PSet Tweak
                cacheUrl = t.data.application.configuration.configCacheUrl
                cacheDb  = t.data.application.configuration.cacheName
                configId = t.data.application.configuration.configId
                tweakTarget = t.data.application.command.psetTweak

                configCache = ConfigCache(cacheUrl, cacheDb)
                configCache.loadByID(configId)
                configCache.saveConfigToDisk(targetFile = fileTarget)
                tweak = TweakAPI.makeTweakFromJSON(configCache.getPSetTweaks())
                if tweak:
                    tweakFile = "%s/%s" % (stepPath, tweakTarget)
                    tweak.persist(tweakFile, "json")

# 需要导入模块: from WMCore.Cache.WMConfigCache import ConfigCache [as 别名]
# 或者: from WMCore.Cache.WMConfigCache.ConfigCache import loadByID [as 别名]
    def testB_addingConfigsAndTweaks(self):
        """
        _addingConfigsAndTweaks_
        
        Test adding config files and tweak files
        """
        PSetTweak = "Hello, I am a PSetTweak.  It's nice to meet you."
        attach    = "Hello, I am an attachment"

        configCache = ConfigCache(os.environ["COUCHURL"], couchDBName = 'config_test')
        configCache.createUserGroup(groupname = "testGroup", username = 'testOps')
        configCache.setPSetTweaks(PSetTweak = PSetTweak)
        configCache.attachments['attach1'] = attach
        psetPath = os.path.join(getTestBase(), "WMCore_t/Cache_t/PSet.txt")
        configCache.addConfig(newConfig = psetPath, psetHash = None)

        configCache.setLabel("sample-label")
        configCache.setDescription("describe this config here")
        configCache.save()
        configString1 = configCache.getConfig()

        configCache2 = ConfigCache(os.environ["COUCHURL"], couchDBName = 'config_test',
                                   id = configCache.getCouchID(),
                                   rev = configCache.getCouchRev())
        configCache2.loadByID(configCache.getCouchID())
        configString2 = configCache2.getConfig()
        
        self.assertEqual(configString1, configString2)
        self.assertEqual(configCache2.attachments.get('attach1', None), attach)

        configCache.delete()
        return

# 需要导入模块: from WMCore.Cache.WMConfigCache import ConfigCache [as 别名]
# 或者: from WMCore.Cache.WMConfigCache.ConfigCache import loadByID [as 别名]
def createAlgoFromInfo(info):
    """
    Create an Algo object from basic information

    """
    
    algo = {'ApplicationName':    info.get('ApplicationName'),
            'ApplicationFamily':  info.get('ApplicationFamily'),
            'ApplicationVersion': info.get('ApplicationVersion'),
            'PSetHash':           info.get('PSetHash'),
            'PSetContent':        None,
            'InDBS':              info.get('AlgoInDBS', None)
            }

    configString = info.get('PSetContent')
    if configString:
        split = configString.split(';;')
        cacheURL = split[0]
        cacheDB  = split[1]
        configID = split[2]
        try:
            configCache = ConfigCache(cacheURL, cacheDB)
            configCache.loadByID(configID)
            algo['PSetContent'] = configCache.getConfig()
        except Exception, ex:
            msg =  "Exception in getting configCache from DB\n"
            msg += "Ignoring this exception and continuing without config.\n"
            msg += str(ex)
            msg += str(traceback.format_exc())
            logging.error(msg)
            logging.debug("URL: %s,  DB: %s,  ID: %s" % (cacheURL, cacheDB, configID))

# 需要导入模块: from WMCore.Cache.WMConfigCache import ConfigCache [as 别名]
# 或者: from WMCore.Cache.WMConfigCache.ConfigCache import loadByID [as 别名]
 def showOriginalConfig(self, docId):
     """ Makes a link to the original text of the config """
     configCache = ConfigCache(self.couchUrl, self.configDBName)
     configCache.loadByID(docId)
     configString =  configCache.getConfig()
     if configString == None:
         return "Cannot find document " + str(docId) + " in Couch DB"
     return '<pre>' + configString + '</pre>'

# 需要导入模块: from WMCore.Cache.WMConfigCache import ConfigCache [as 别名]
# 或者: from WMCore.Cache.WMConfigCache.ConfigCache import loadByID [as 别名]
    def determineOutputModules(self, scenarioFunc = None, scenarioArgs = None,
                               configDoc = None, couchURL = None,
                               couchDBName = None, configCacheUrl = None):
        """
        _determineOutputModules_

        Determine the output module names and associated metadata for the
        given config.
        """
        # set default scenarioArgs to empty dictionary if it is None.
        scenarioArgs = scenarioArgs or {}

        outputModules = {}
        if configDoc != None and configDoc != "":
            url = configCacheUrl or couchURL
            if  (url, couchDBName) in self.config_cache:
                configCache = self.config_cache[(url, couchDBName)]
            else:
                configCache = ConfigCache(url, couchDBName, True)
                self.config_cache[(url, couchDBName)] = configCache
            #TODO: need to change to DataCache
            #configCache.loadDocument(configDoc)
            configCache.loadByID(configDoc)
            outputModules = configCache.getOutputModuleInfo()
        else:
            if 'outputs' in scenarioArgs and scenarioFunc in [ "promptReco", "expressProcessing", "repack" ]:
                for output in scenarioArgs.get('outputs', []):
                    moduleLabel = output['moduleLabel']
                    outputModules[moduleLabel] = { 'dataTier' : output['dataTier'] }
                    if 'primaryDataset' in output:
                        outputModules[moduleLabel]['primaryDataset'] = output['primaryDataset']
                    if 'filterName' in output:
                        outputModules[moduleLabel]['filterName'] = output['filterName']

            elif 'writeTiers' in scenarioArgs and scenarioFunc == "promptReco":
                for dataTier in scenarioArgs.get('writeTiers'):
                    moduleLabel = "%soutput" % dataTier
                    outputModules[moduleLabel] = { 'dataTier' : dataTier }

            elif scenarioFunc == "alcaSkim":
                for alcaSkim in scenarioArgs.get('skims',[]):
                    moduleLabel = "ALCARECOStream%s" % alcaSkim
                    if alcaSkim.startswith("PromptCalibProd"):
                        dataTier = "ALCAPROMPT"
                    else:
                        dataTier = "ALCARECO"
                    outputModules[moduleLabel] = { 'dataTier' : dataTier,
                                                   'primaryDataset' : scenarioArgs.get('primaryDataset'),
                                                   'filterName' : alcaSkim }

        return outputModules

# 需要导入模块: from WMCore.Cache.WMConfigCache import ConfigCache [as 别名]
# 或者: from WMCore.Cache.WMConfigCache.ConfigCache import loadByID [as 别名]
 def _getConfigCache(self, requestName, processMethod):
     try:
         request = Utilities.requestDetails(requestName)
     except Exception as ex:
         msg = "Cannot find request %s, check logs." % requestName
         logging.error("%s, reason: %s" % (msg, ex))
         return msg
     url = request.get("ConfigCacheUrl", None) or self.couchUrl
     try:
         configCache = ConfigCache(url, self.configDBName)
         configDocId = request["ConfigCacheID"]
         configCache.loadByID(configDocId)
     except Exception as ex:
         msg = "Cannot find ConfigCache document %s on %s." % (configDocId, url)
         logging.error("%s, reason: %s" % (msg, ex))
         return msg
     return getattr(configCache, processMethod)()

# 需要导入模块: from WMCore.Cache.WMConfigCache import ConfigCache [as 别名]
# 或者: from WMCore.Cache.WMConfigCache.ConfigCache import loadByID [as 别名]
def createAlgoFromInfo(info):
    """
    Create an Algo object from basic information

    """

    algo = {
        "ApplicationName": info["ApplicationName"],
        "ApplicationFamily": info["ApplicationFamily"],
        "ApplicationVersion": info["ApplicationVersion"],
        "PSetHash": info["PSetHash"],
        "PSetContent": None,
        "InDBS": info["AlgoInDBS"],
    }

    configString = info.get("PSetContent")
    if configString:
        try:
            split = configString.split(";;")
            cacheURL = split[0]
            cacheDB = split[1]
            configID = split[2]
        except IndexError:
            msg = "configCache not properly formatted\n"
            msg += "configString\n: %s" % configString
            msg += "Not attempting to put configCache content in DBS for this algo"
            msg += "AlgoInfo: %s" % algo
            logging.error(msg)
            return algo
        if cacheURL == "None" or cacheDB == "None" or configID == "None":
            # No Config for this DB
            logging.debug("No configCache for this algo")
            return algo
        try:
            configCache = ConfigCache(cacheURL, cacheDB)
            configCache.loadByID(configID)
            algo["PSetContent"] = configCache.getConfig()
        except Exception as ex:
            msg = "Exception in getting configCache from DB\n"
            msg += "Ignoring this exception and continuing without config.\n"
            msg += str(ex)
            msg += str(traceback.format_exc())
            logging.error(msg)
            logging.debug("URL: %s,  DB: %s,  ID: %s" % (cacheURL, cacheDB, configID))

    return algo

# 需要导入模块: from WMCore.Cache.WMConfigCache import ConfigCache [as 别名]
# 或者: from WMCore.Cache.WMConfigCache.ConfigCache import loadByID [as 别名]
    def validateConfigCacheExists(self, configID, couchURL, couchDBName,
                                  getOutputModules = False):
        """
        _validateConfigCacheExists_

        If we have a configCache, we should probably try and load it.
        """

        if configID == '' or configID == ' ':
            self.raiseValidationException(msg = "ConfigCacheID is invalid and cannot be loaded")

        configCache = ConfigCache(dbURL = couchURL, couchDBName = couchDBName,
                                  id = configID)
        try:
            configCache.loadByID(configID = configID)
        except ConfigCacheException:
            self.raiseValidationException(msg = "Failure to load ConfigCache while validating workload")

        duplicateCheck = {}
        try:
            outputModuleInfo = configCache.getOutputModuleInfo()
        except Exception:
            # Something's gone wrong with trying to open the configCache
            msg = "Error in getting output modules from ConfigCache during workload validation.  Check ConfigCache formatting!"
            self.raiseValidationException(msg = msg)
        for outputModule in outputModuleInfo.values():
            dataTier   = outputModule.get('dataTier', None)
            filterName = outputModule.get('filterName', None)
            if not dataTier:
                self.raiseValidationException(msg = "No DataTier in output module.")

            # Add dataTier to duplicate dictionary
            if not dataTier in duplicateCheck.keys():
                duplicateCheck[dataTier] = []
            if filterName in duplicateCheck[dataTier]:
                # Then we've seen this combination before
                self.raiseValidationException(msg = "Duplicate dataTier/filterName combination.")
            else:
                duplicateCheck[dataTier].append(filterName)

        if getOutputModules:
            return outputModuleInfo

        return

# 需要导入模块: from WMCore.Cache.WMConfigCache import ConfigCache [as 别名]
# 或者: from WMCore.Cache.WMConfigCache.ConfigCache import loadByID [as 别名]
    def determineOutputModules(self, scenarioFunc = None, scenarioArgs = None,
                               configDoc = None, couchURL = None,
                               couchDBName = None, configCacheUrl = None):
        """
        _determineOutputModules_

        Determine the output module names and associated metadata for the
        given config.
        """
        outputModules = {}
        if configDoc != None and configDoc != "":
            url = configCacheUrl or couchURL
            configCache = ConfigCache(url, couchDBName)
            configCache.loadByID(configDoc)
            outputModules = configCache.getOutputModuleInfo()
        else:
            if 'outputs' in scenarioArgs and scenarioFunc in [ "promptReco", "expressProcessing", "repack" ]:
                for output in scenarioArgs.get('outputs', []):
                    moduleLabel = output['moduleLabel']
                    outputModules[moduleLabel] = { 'dataTier' : output['dataTier'] }
                    if output.has_key('primaryDataset'):
                        outputModules[moduleLabel]['primaryDataset'] = output['primaryDataset']
                    if output.has_key('filterName'):
                        outputModules[moduleLabel]['filterName'] = output['filterName']

            elif 'writeTiers' in scenarioArgs and scenarioFunc == "promptReco":
                for dataTier in scenarioArgs.get('writeTiers'):
                    moduleLabel = "%soutput" % dataTier
                    outputModules[moduleLabel] = { 'dataTier' : dataTier }

            elif scenarioFunc == "alcaSkim":
                for alcaSkim in scenarioArgs.get('skims',[]):
                    moduleLabel = "ALCARECOStream%s" % alcaSkim
                    if alcaSkim == "PromptCalibProd":
                        dataTier = "ALCAPROMPT"
                    else:
                        dataTier = "ALCARECO"
                    outputModules[moduleLabel] = { 'dataTier' : dataTier,
                                                   'primaryDataset' : scenarioArgs.get('primaryDataset'),
                                                   'filterName' : alcaSkim }

        return outputModules

# 需要导入模块: from WMCore.Cache.WMConfigCache import ConfigCache [as 别名]
# 或者: from WMCore.Cache.WMConfigCache.ConfigCache import loadByID [as 别名]
    def determineOutputModules(self, scenarioName = None, scenarioArgs = None,
                               configDoc = None, couchURL = None,
                               couchDBName = None):
        """
        _determineOutputModules_

        Determine the output module names and associated metadata for the
        given config.
        """
        outputModules = {}
        if configDoc != None and configDoc != "":
            configCache = ConfigCache(couchURL, couchDBName)
            configCache.loadByID(configDoc)
            outputModules = configCache.getOutputModuleInfo()
        else:
            for dataTier in scenarioArgs.get("writeTiers",[]):
                outputModuleName = "output%s%s" % (dataTier, dataTier)
                outputModules[outputModuleName] = {"dataTier": dataTier,
                                                   "filterName": None}

        return outputModules

# 需要导入模块: from WMCore.Cache.WMConfigCache import ConfigCache [as 别名]
# 或者: from WMCore.Cache.WMConfigCache.ConfigCache import loadByID [as 别名]
    def testA_basicConfig(self):
        """
        _basicConfig_

        Basic configCache stuff.
        """
        PSetTweak = "Hello, I am a PSetTweak.  It's nice to meet you."

        configCache = ConfigCache(os.environ["COUCHURL"], couchDBName = 'config_test')
        configCache.createUserGroup(groupname = "testGroup", username = 'testOps')
        configCache.setPSetTweaks(PSetTweak = PSetTweak)
        configCache.save()

        configCache2 = ConfigCache(os.environ["COUCHURL"], couchDBName = 'config_test',
                                   id = configCache.getCouchID(),
                                   rev = configCache.getCouchRev())
        configCache2.loadByID(configCache.getCouchID())

        self.assertEqual(configCache2.getPSetTweaks(), PSetTweak)

        configCache2.delete()
        configCache3 = ConfigCache(os.environ["COUCHURL"], couchDBName = 'config_test',
                                   id = configCache.getCouchID(),
                                   rev = configCache.getCouchRev())

        testFlag = False

        # It should fail to load deleted documents
        try:
            configCache3.loadByID(configCache.getCouchID())
        except ConfigCacheException:
            testFlag = True

        self.assertTrue(testFlag)

        return

# 需要导入模块: from WMCore.Cache.WMConfigCache import ConfigCache [as 别名]
# 或者: from WMCore.Cache.WMConfigCache.ConfigCache import loadByID [as 别名]
 def showTweakFile(self, docId):
     """ Makes a link to the dump of the tweakfile """
     configCache = ConfigCache(self.couchUrl, self.configDBName)
     configCache.loadByID(docId)
     return str(configCache.getPSetTweaks()).replace('\n', '<br>')

# 需要导入模块: from WMCore.Cache.WMConfigCache import ConfigCache [as 别名]
# 或者: from WMCore.Cache.WMConfigCache.ConfigCache import loadByID [as 别名]
                                messages=request['RequestMessages'],
                                updateDictList=request['RequestUpdates'])
        
        
    def _getConfigCache(self, requestName, processMethod):
        try:
            request = Utilities.requestDetails(requestName)
        except Exception, ex:
            msg = "Cannot find request %s, check logs." % requestName
            logging.error("%s, reason: %s" % (msg, ex))
            return msg
        url = request.get("ConfigCacheUrl", None) or self.couchUrl
        try:
            configCache = ConfigCache(url, self.configDBName)
            configDocId = request["ConfigCacheID"]
            configCache.loadByID(configDocId)
        except Exception, ex:
            msg = "Cannot find ConfigCache document %s on %s." % (configDocId, url)
            logging.error("%s, reason: %s" % (msg, ex))
            return msg
        return getattr(configCache, processMethod)()
            

    @cherrypy.expose
    @cherrypy.tools.secmodv2()
    def showOriginalConfig(self, requestName):
        """
        Makes a link to the original text of the config document.
        
        """
        self.validate(requestName)

# 需要导入模块: from WMCore.Cache.WMConfigCache import ConfigCache [as 别名]
# 或者: from WMCore.Cache.WMConfigCache.ConfigCache import loadByID [as 别名]
    def determineOutputModules(
        self, scenarioFunc=None, scenarioArgs=None, configDoc=None, couchURL=None, couchDBName=None, configCacheUrl=None
    ):
        """
        _determineOutputModules_

        Determine the output module names and associated metadata for the
        given config.
        """
        # set default scenarioArgs to empty dictionary if it is None.
        scenarioArgs = scenarioArgs or {}

        outputModules = {}
        if configDoc != None and configDoc != "":
            url = configCacheUrl or couchURL
            if (url, couchDBName) in self.config_cache:
                configCache = self.config_cache[(url, couchDBName)]
            else:
                configCache = ConfigCache(url, couchDBName, True)
                self.config_cache[(url, couchDBName)] = configCache
            # TODO: need to change to DataCache
            # configCache.loadDocument(configDoc)
            configCache.loadByID(configDoc)
            outputModules = configCache.getOutputModuleInfo()
        else:
            if "outputs" in scenarioArgs and scenarioFunc in ["promptReco", "expressProcessing", "repack"]:

                for output in scenarioArgs.get("outputs", []):

                    moduleLabel = output["moduleLabel"]
                    outputModules[moduleLabel] = {"dataTier": output["dataTier"]}
                    if "primaryDataset" in output:
                        outputModules[moduleLabel]["primaryDataset"] = output["primaryDataset"]
                    if "filterName" in output:
                        outputModules[moduleLabel]["filterName"] = output["filterName"]

                for physicsSkim in scenarioArgs.get("PhysicsSkims", []):
                    skimToDataTier = {
                        "LogError": "RAW-RECO",
                        "LogErrorMonitor": "USER",
                        "ZElectron": "RAW-RECO",
                        "ZMu": "RAW-RECO",
                        "MuTau": "RAW-RECO",
                        "TopMuEG": "RAW-RECO",
                        "EcalActivity": "RAW-RECO",
                        "CosmicSP": "RAW-RECO",
                        "CosmicTP": "RAW-RECO",
                        "ZMM": "RAW-RECO",
                        "Onia": "RECO",
                        "HighPtJet": "RAW-RECO",
                        "D0Meson": "RECO",
                        "Photon": "AOD",
                        "ZEE": "AOD",
                        "BJet": "AOD",
                        "OniaCentral": "RECO",
                        "OniaPeripheral": "RECO",
                        "SingleTrack": "AOD",
                        "MinBias": "AOD",
                        "OniaUPC": "RAW-RECO",
                        "HighMET": "RECO",
                        "BPHSkim": "USER",
                    }
                    dataTier = skimToDataTier.get(physicsSkim, "USER")
                    moduleLabel = "SKIMStream%s" % physicsSkim
                    outputModules[moduleLabel] = {"dataTier": dataTier, "filterName": physicsSkim}

            elif scenarioFunc == "alcaSkim":

                for alcaSkim in scenarioArgs.get("skims", []):
                    moduleLabel = "ALCARECOStream%s" % alcaSkim
                    if alcaSkim.startswith("PromptCalibProd"):
                        dataTier = "ALCAPROMPT"
                    else:
                        dataTier = "ALCARECO"
                    outputModules[moduleLabel] = {
                        "dataTier": dataTier,
                        "primaryDataset": scenarioArgs.get("primaryDataset"),
                        "filterName": alcaSkim,
                    }

        return outputModules