Python WMConfigCache.ConfigCache类代码示例

def createAlgoFromInfo(info):
    """
    Create an Algo object from basic information

    """
    
    algo = {'ApplicationName':    info.get('ApplicationName'),
            'ApplicationFamily':  info.get('ApplicationFamily'),
            'ApplicationVersion': info.get('ApplicationVersion'),
            'PSetHash':           info.get('PSetHash'),
            'PSetContent':        None,
            'InDBS':              info.get('AlgoInDBS', None)
            }

    configString = info.get('PSetContent')
    if configString:
        split = configString.split(';;')
        cacheURL = split[0]
        cacheDB  = split[1]
        configID = split[2]
        try:
            configCache = ConfigCache(cacheURL, cacheDB)
            configCache.loadByID(configID)
            algo['PSetContent'] = configCache.getConfig()
        except Exception, ex:
            msg =  "Exception in getting configCache from DB\n"
            msg += "Ignoring this exception and continuing without config.\n"
            msg += str(ex)
            msg += str(traceback.format_exc())
            logging.error(msg)
            logging.debug("URL: %s,  DB: %s,  ID: %s" % (cacheURL, cacheDB, configID))

    def createConfig(self, bad=False):
        """
        _createConfig_

        Create a config of some sort that we can load out of ConfigCache
        """

        PSetTweak = {
            "process": {
                "outputModules_": ["ThisIsAName"],
                "ThisIsAName": {"dataset": {"dataTier": "RECO", "filterName": "Filter"}},
            }
        }

        BadTweak = {
            "process": {
                "outputModules_": ["ThisIsAName1", "ThisIsAName2"],
                "ThisIsAName1": {"dataset": {"dataTier": "RECO", "filterName": "Filter"}},
                "ThisIsAName2": {"dataset": {"dataTier": "RECO", "filterName": "Filter"}},
            }
        }

        configCache = ConfigCache(os.environ["COUCHURL"], couchDBName=self.couchDBName)
        configCache.createUserGroup(groupname="testGroup", username="testOps")
        if bad:
            configCache.setPSetTweaks(PSetTweak=BadTweak)
        else:
            configCache.setPSetTweaks(PSetTweak=PSetTweak)
        configCache.save()

        return configCache.getCouchID()

    def __call__(self, wmTask):
        """
        Trip through steps, find CMSSW steps, pull in config files,
        PSet Tweaks etc

        """
        for t in wmTask.steps().nodeIterator():
            t = WMStep.WMStepHelper(t)
            stepPath = "%s/%s" % (self.workingDirectory(), t.name())

            # the CMSSW has a special case with its ConfigCache argument
            if not t.stepType() in ("CMSSW", "MulticoreCMSSW"): continue
            if (hasattr(t.data.application.configuration,'retrieveConfigUrl')):
                # main config file
                fileTarget = "%s/%s" % (
                    stepPath,
                    t.data.application.command.configuration)
                urllib.urlretrieve(
                    t.data.application.configuration.retrieveConfigUrl,
                    fileTarget)
                # PSet Tweak
                cacheUrl = t.data.application.configuration.configCacheUrl
                cacheDb  = t.data.application.configuration.cacheName
                configId = t.data.application.configuration.configId
                tweakTarget = t.data.application.command.psetTweak

                configCache = ConfigCache(cacheUrl, cacheDb)
                configCache.loadByID(configId)
                tweak = TweakAPI.makeTweakFromJSON(configCache.getPSetTweaks())
                if tweak:
                    tweakFile = "%s/%s" % (stepPath, tweakTarget)
                    tweak.persist(tweakFile, "json")

 def showOriginalConfig(self, docId):
     """ Makes a link to the original text of the config """
     configCache = ConfigCache(self.couchUrl, self.configDBName)
     configCache.loadByID(docId)
     configString =  configCache.getConfig()
     if configString == None:
         return "Cannot find document " + str(docId) + " in Couch DB"
     return '<pre>' + configString + '</pre>'

    def __init__(self, **options):
        GeneratorInterface.__init__(self, **options)
        self.couchUrl = options.get("CouchUrl")
        self.couchDBName = options.get("CouchDBName")
        self.couchConfigDoc = options.get("ConfigCacheDoc")

        confCache = ConfigCache(dbURL = self.couchUrl, couchDBName = self.couchDBName, id = self.couchConfigDoc)
        confCache.load()
        seeds = confCache.document[u'pset_tweak_details'][u'process'][u'RandomNumberGeneratorService']
        self.seedTable = []
        for k in seeds.keys():
            if k == u"parameters_" : continue
            self.seedTable.append("process.RandomNumberGeneratorService.%s.initialSeed" % k)

    def determineOutputModules(self, scenarioFunc = None, scenarioArgs = None,
                               configDoc = None, couchURL = None,
                               couchDBName = None, configCacheUrl = None):
        """
        _determineOutputModules_

        Determine the output module names and associated metadata for the
        given config.
        """
        # set default scenarioArgs to empty dictionary if it is None.
        scenarioArgs = scenarioArgs or {}

        outputModules = {}
        if configDoc != None and configDoc != "":
            url = configCacheUrl or couchURL
            if  (url, couchDBName) in self.config_cache:
                configCache = self.config_cache[(url, couchDBName)]
            else:
                configCache = ConfigCache(url, couchDBName, True)
                self.config_cache[(url, couchDBName)] = configCache
            #TODO: need to change to DataCache
            #configCache.loadDocument(configDoc)
            configCache.loadByID(configDoc)
            outputModules = configCache.getOutputModuleInfo()
        else:
            if 'outputs' in scenarioArgs and scenarioFunc in [ "promptReco", "expressProcessing", "repack" ]:
                for output in scenarioArgs.get('outputs', []):
                    moduleLabel = output['moduleLabel']
                    outputModules[moduleLabel] = { 'dataTier' : output['dataTier'] }
                    if 'primaryDataset' in output:
                        outputModules[moduleLabel]['primaryDataset'] = output['primaryDataset']
                    if 'filterName' in output:
                        outputModules[moduleLabel]['filterName'] = output['filterName']

            elif 'writeTiers' in scenarioArgs and scenarioFunc == "promptReco":
                for dataTier in scenarioArgs.get('writeTiers'):
                    moduleLabel = "%soutput" % dataTier
                    outputModules[moduleLabel] = { 'dataTier' : dataTier }

            elif scenarioFunc == "alcaSkim":
                for alcaSkim in scenarioArgs.get('skims',[]):
                    moduleLabel = "ALCARECOStream%s" % alcaSkim
                    if alcaSkim.startswith("PromptCalibProd"):
                        dataTier = "ALCAPROMPT"
                    else:
                        dataTier = "ALCARECO"
                    outputModules[moduleLabel] = { 'dataTier' : dataTier,
                                                   'primaryDataset' : scenarioArgs.get('primaryDataset'),
                                                   'filterName' : alcaSkim }

        return outputModules

def createAlgoFromInfo(info):
    """
    Create an Algo object from basic information

    """

    algo = {
        "ApplicationName": info["ApplicationName"],
        "ApplicationFamily": info["ApplicationFamily"],
        "ApplicationVersion": info["ApplicationVersion"],
        "PSetHash": info["PSetHash"],
        "PSetContent": None,
        "InDBS": info["AlgoInDBS"],
    }

    configString = info.get("PSetContent")
    if configString:
        try:
            split = configString.split(";;")
            cacheURL = split[0]
            cacheDB = split[1]
            configID = split[2]
        except IndexError:
            msg = "configCache not properly formatted\n"
            msg += "configString\n: %s" % configString
            msg += "Not attempting to put configCache content in DBS for this algo"
            msg += "AlgoInfo: %s" % algo
            logging.error(msg)
            return algo
        if cacheURL == "None" or cacheDB == "None" or configID == "None":
            # No Config for this DB
            logging.debug("No configCache for this algo")
            return algo
        try:
            configCache = ConfigCache(cacheURL, cacheDB)
            configCache.loadByID(configID)
            algo["PSetContent"] = configCache.getConfig()
        except Exception as ex:
            msg = "Exception in getting configCache from DB\n"
            msg += "Ignoring this exception and continuing without config.\n"
            msg += str(ex)
            msg += str(traceback.format_exc())
            logging.error(msg)
            logging.debug("URL: %s,  DB: %s,  ID: %s" % (cacheURL, cacheDB, configID))

    return algo

 def _getConfigCache(self, requestName, processMethod):
     try:
         request = Utilities.requestDetails(requestName)
     except Exception as ex:
         msg = "Cannot find request %s, check logs." % requestName
         logging.error("%s, reason: %s" % (msg, ex))
         return msg
     url = request.get("ConfigCacheUrl", None) or self.couchUrl
     try:
         configCache = ConfigCache(url, self.configDBName)
         configDocId = request["ConfigCacheID"]
         configCache.loadByID(configDocId)
     except Exception as ex:
         msg = "Cannot find ConfigCache document %s on %s." % (configDocId, url)
         logging.error("%s, reason: %s" % (msg, ex))
         return msg
     return getattr(configCache, processMethod)()

    def __call__(self, workloadName, arguments):
        """
        _call_

        Create a PromptSkimming workload with the given parameters.
        """
        self.injectIntoConfigCache(arguments["CMSSWVersion"], arguments["ScramArch"],
                                   arguments["InitCommand"], arguments["SkimConfig"], workloadName,
                                   arguments["CouchURL"], arguments["CouchDBName"])

        configCache = ConfigCache(arguments["CouchURL"], arguments["CouchDBName"])
        arguments["ProcConfigCacheID"] = configCache.getIDFromLabel(workloadName)
        
        workload = DataProcessingWorkloadFactory.__call__(self, workloadName, arguments)
        workload.setSiteWhitelist(arguments["CustodialSite"])
        workload.setBlockWhitelist(arguments["BlockName"])
        return workload

    def setUp(self):
        """
        _setUp_
        
        setUp function for unittest

        """
        # Set constants
        self.couchDB      = "config_test"
        self.configURL    = "RANDOM;;URL;;NAME"
        self.configString = "This is a random string"
        
        self.testInit = TestInit(__file__)
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()
        self.testInit.setSchema(customModules = 
                                ["WMComponent.DBS3Buffer",
                                 'WMCore.Agent.Database'],
                                useDefault = False)
        self.testInit.setupCouch(self.couchDB, "GroupUser", "ConfigCache")
      
        myThread = threading.currentThread()
        self.bufferFactory = DAOFactory(package = "WMComponent.DBSBuffer.Database",
                                        logger = myThread.logger,
                                        dbinterface = myThread.dbi)

        locationAction = self.bufferFactory(classname = "DBSBufferFiles.AddLocation")
        locationAction.execute(siteName = "se1.cern.ch")
        locationAction.execute(siteName = "se1.fnal.gov")
        locationAction.execute(siteName = "malpaquet") 


        # Set heartbeat
        self.componentName = 'JobSubmitter'
        self.heartbeatAPI  = HeartbeatAPI(self.componentName)
        self.heartbeatAPI.registerComponent()

        # Set up a config cache
        configCache = ConfigCache(os.environ["COUCHURL"], couchDBName = self.couchDB)
        configCache.createUserGroup(groupname = "testGroup", username = 'testOps')
        self.testDir = self.testInit.generateWorkDir()

        psetPath = os.path.join(self.testDir, "PSet.txt")
        f = open(psetPath, 'w')
        f.write(self.configString)
        f.close()
        
        configCache.addConfig(newConfig = psetPath, psetHash = None)
        configCache.save()
        self.configURL = "%s;;%s;;%s" % (os.environ["COUCHURL"],
                                         self.couchDB,
                                         configCache.getCouchID())

        return

    def validateConfigCacheExists(self, configID, couchURL, couchDBName,
                                  getOutputModules = False):
        """
        _validateConfigCacheExists_

        If we have a configCache, we should probably try and load it.
        """

        if configID == '' or configID == ' ':
            self.raiseValidationException(msg = "ConfigCacheID is invalid and cannot be loaded")

        configCache = ConfigCache(dbURL = couchURL, couchDBName = couchDBName,
                                  id = configID)
        try:
            configCache.loadByID(configID = configID)
        except ConfigCacheException:
            self.raiseValidationException(msg = "Failure to load ConfigCache while validating workload")

        duplicateCheck = {}
        try:
            outputModuleInfo = configCache.getOutputModuleInfo()
        except Exception:
            # Something's gone wrong with trying to open the configCache
            msg = "Error in getting output modules from ConfigCache during workload validation.  Check ConfigCache formatting!"
            self.raiseValidationException(msg = msg)
        for outputModule in outputModuleInfo.values():
            dataTier   = outputModule.get('dataTier', None)
            filterName = outputModule.get('filterName', None)
            if not dataTier:
                self.raiseValidationException(msg = "No DataTier in output module.")

            # Add dataTier to duplicate dictionary
            if not dataTier in duplicateCheck.keys():
                duplicateCheck[dataTier] = []
            if filterName in duplicateCheck[dataTier]:
                # Then we've seen this combination before
                self.raiseValidationException(msg = "Duplicate dataTier/filterName combination.")
            else:
                duplicateCheck[dataTier].append(filterName)

        if getOutputModules:
            return outputModuleInfo

        return

    def determineOutputModules(self, scenarioFunc = None, scenarioArgs = None,
                               configDoc = None, couchURL = None,
                               couchDBName = None, configCacheUrl = None):
        """
        _determineOutputModules_

        Determine the output module names and associated metadata for the
        given config.
        """
        outputModules = {}
        if configDoc != None and configDoc != "":
            url = configCacheUrl or couchURL
            configCache = ConfigCache(url, couchDBName)
            configCache.loadByID(configDoc)
            outputModules = configCache.getOutputModuleInfo()
        else:
            if 'outputs' in scenarioArgs and scenarioFunc in [ "promptReco", "expressProcessing", "repack" ]:
                for output in scenarioArgs.get('outputs', []):
                    moduleLabel = output['moduleLabel']
                    outputModules[moduleLabel] = { 'dataTier' : output['dataTier'] }
                    if output.has_key('primaryDataset'):
                        outputModules[moduleLabel]['primaryDataset'] = output['primaryDataset']
                    if output.has_key('filterName'):
                        outputModules[moduleLabel]['filterName'] = output['filterName']

            elif 'writeTiers' in scenarioArgs and scenarioFunc == "promptReco":
                for dataTier in scenarioArgs.get('writeTiers'):
                    moduleLabel = "%soutput" % dataTier
                    outputModules[moduleLabel] = { 'dataTier' : dataTier }

            elif scenarioFunc == "alcaSkim":
                for alcaSkim in scenarioArgs.get('skims',[]):
                    moduleLabel = "ALCARECOStream%s" % alcaSkim
                    if alcaSkim == "PromptCalibProd":
                        dataTier = "ALCAPROMPT"
                    else:
                        dataTier = "ALCARECO"
                    outputModules[moduleLabel] = { 'dataTier' : dataTier,
                                                   'primaryDataset' : scenarioArgs.get('primaryDataset'),
                                                   'filterName' : alcaSkim }

        return outputModules

    def __call__(self, workloadName, arguments):
        """
        _call_

        Create a PromptSkimming workload with the given parameters.
        """
        configCouchUrl = arguments.get("ConfigCacheUrl", None) or arguments["CouchURL"]
        injectIntoConfigCache(arguments["CMSSWVersion"], arguments["ScramArch"],
                              arguments["InitCommand"], arguments["SkimConfig"], workloadName,
                              configCouchUrl, arguments["CouchDBName"],
                              arguments.get("EnvPath", None), arguments.get("BinPath", None))

        try:
            configCache = ConfigCache(configCouchUrl, arguments["CouchDBName"])
            arguments["ConfigCacheID"] = configCache.getIDFromLabel(workloadName)
            if not arguments["ConfigCacheID"]:
                logging.error("The configuration was not uploaded to couch")
                raise Exception
        except Exception:
            logging.error("There was an exception loading the config out of the")
            logging.error("ConfigCache.  Check the scramOutput.log file in the")
            logging.error("PromptSkimScheduler directory to find out what went")
            logging.error("wrong.")
            raise

        parsedProcVer = parseT0ProcVer(arguments["ProcessingVersion"],
                                       'PromptSkim')
        arguments["ProcessingString"] = parsedProcVer["ProcString"]
        arguments["ProcessingVersion"] = parsedProcVer["ProcVer"]

        workload = DataProcessingWorkloadFactory.__call__(self, workloadName, arguments)

        # We need to strip off "MSS" as that causes all sorts of problems.
        if arguments["CustodialSite"].find("MSS") != -1:
            site = arguments["CustodialSite"][:-4]
        else:
            site = arguments["CustodialSite"]

        workload.setSiteWhitelist(site)
        workload.setBlockWhitelist(arguments["BlockName"])
        return workload

    def determineOutputModules(self, scenarioName = None, scenarioArgs = None,
                               configDoc = None, couchURL = None,
                               couchDBName = None):
        """
        _determineOutputModules_

        Determine the output module names and associated metadata for the
        given config.
        """
        outputModules = {}
        if configDoc != None and configDoc != "":
            configCache = ConfigCache(couchURL, couchDBName)
            configCache.loadByID(configDoc)
            outputModules = configCache.getOutputModuleInfo()
        else:
            for dataTier in scenarioArgs.get("writeTiers",[]):
                outputModuleName = "output%s%s" % (dataTier, dataTier)
                outputModules[outputModuleName] = {"dataTier": dataTier,
                                                   "filterName": None}

        return outputModules

    def validateConfigCacheExists(self, configID, couchURL, couchDBName, getOutputModules=True):
        """
        _validateConfigCacheExists_

        If we have a configCache, we should probably try and load it.
        """

        if configID == "" or configID == " ":
            self.raiseValidationException(msg="ConfigCacheID is invalid and cannot be loaded")

        if (couchURL, couchDBName) in self.config_cache:
            configCache = self.config_cache[(couchURL, couchDBName)]
        else:
            configCache = ConfigCache(dbURL=couchURL, couchDBName=couchDBName, detail=getOutputModules)
            self.config_cache[(couchURL, couchDBName)] = configCache

        try:
            # if dtail option is set return outputModules
            return configCache.validate(configID)
        except ConfigCacheException as ex:
            self.raiseValidationException(ex.message())