本文整理汇总了Python中WMCore.WMSpec.StdSpecs.DataProcessing.DataProcessing类的典型用法代码示例。如果您正苦于以下问题：Python DataProcessing类的具体用法？Python DataProcessing怎么用？Python DataProcessing使用的例子？那么恭喜您, 这里精选的类代码示例或许可以为您提供帮助。
在下文中一共展示了DataProcessing类的15个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: __call__
def __call__(self, workloadName, arguments):
"""
_call_
Create a ReReco workload with the given parameters.
"""
DataProcessing.__call__(self, workloadName, arguments)
# Arrange the skims in a skimConfig object (i.e. a list of skim configurations)
self.skimConfigs = []
skimIndex = 1
while "SkimName%s" % skimIndex in arguments:
skimConfig = {}
skimConfig["SkimName"] = arguments["SkimName%s" % skimIndex]
skimConfig["SkimInput"] = arguments["SkimInput%s" % skimIndex]
skimConfig["ConfigCacheID"] = arguments["Skim%sConfigCacheID" % skimIndex]
skimConfig["TimePerEvent"] = float(arguments.get("SkimTimePerEvent%s" % skimIndex, self.timePerEvent))
skimConfig["SizePerEvent"] = float(arguments.get("SkimSizePerEvent%s" % skimIndex, self.sizePerEvent))
skimConfig["Memory"] = float(arguments.get("SkimMemory%s" % skimIndex, self.memory))
skimConfig["SkimJobSplitAlgo"] = arguments.get("SkimSplittingAlgo%s" % skimIndex, "FileBased")
skimConfig["SkimJobSplitArgs"] = {"include_parents" : True}
if skimConfig["SkimJobSplitAlgo"] == "FileBased":
skimConfig["SkimJobSplitArgs"]["files_per_job"] = int(arguments.get("SkimFilesPerJob%s" % skimIndex, 1))
elif skimConfig["SkimJobSplitAlgo"] == "EventBased" or skimConfig["SkimJobSplitAlgo"] == "EventAwareLumiBased":
skimConfig["SkimJobSplitArgs"]["events_per_job"] = int(arguments.get("SkimEventsPerJob%s" % skimIndex, int((8.0 * 3600.0) / skimConfig["TimePerEvent"])))
if skimConfig["SkimJobSplitAlgo"] == "EventAwareLumiBased":
skimConfig["SkimJobSplitAlgo"]["max_events_per_lumi"] = 20000
elif skimConfig["SkimJobSplitAlgo"] == "LumiBased":
skimConfig["SkimJobSplitArgs"["lumis_per_job"]] = int(arguments.get("SkimLumisPerJob%s" % skimIndex, 8))
self.skimConfigs.append(skimConfig)
skimIndex += 1
return self.buildWorkload()示例2: getWorkloadCreateArgs
def getWorkloadCreateArgs():
baseArgs = DataProcessing.getWorkloadCreateArgs()
specArgs = {"RequestType": {"default": "ReDigi", "optional": False},
"StepOneOutputModuleName": {"null": True},
"StepTwoOutputModuleName": {"null": True},
"ConfigCacheID": {"optional": True, "null": True},
"StepOneConfigCacheID": {"optional": False, "null": True},
"StepTwoConfigCacheID": {"null": True},
"StepThreeConfigCacheID": {"null": True},
"KeepStepOneOutput": {"default": True, "type": strToBool, "null": False},
"KeepStepTwoOutput": {"default": True, "type": strToBool, "null": False},
"StepTwoTimePerEvent": {"type": float, "null": True,
"validate": lambda x: x > 0},
"StepThreeTimePerEvent": {"type": float, "null": True,
"validate": lambda x: x > 0},
"StepTwoSizePerEvent": {"type": float, "null": True,
"validate": lambda x: x > 0},
"StepThreeSizePerEvent": {"type": float, "null": True,
"validate": lambda x: x > 0},
"StepTwoMemory": {"type": float, "null": True,
"validate": lambda x: x > 0},
"StepThreeMemory": {"type": float, "null": True,
"validate": lambda x: x > 0},
"MCPileup": {"validate": dataset, "attr": "mcPileup", "null": True},
"DataPileup": {"null": True, "validate": dataset},
"DeterministicPileup": {"default": False, "type": strToBool, "null": False}}
baseArgs.update(specArgs)
DataProcessing.setDefaultArgumentsProperty(baseArgs)
return baseArgs示例3: __call__
def __call__(self, workloadName, arguments):
"""
_call_
Create a DQMHarvest workload with the given parameters.
"""
DataProcessing.__call__(self, workloadName, arguments)
self.workload = self.createWorkload()
self.workload.setDashboardActivity("harvesting")
splitArgs = {"runs_per_job": 1}
if self.dqmHarvestUnit == "multiRun":
# then it should result in a single job in the end, very high number of runs
splitArgs['runs_per_job'] = 999999
self.workload.setWorkQueueSplitPolicy("Dataset", "Harvest", splitArgs)
# also creates the logCollect job by default
self.addDQMHarvestTask(uploadProxy=self.dqmUploadProxy,
periodic_harvest_interval=self.periodicHarvestInterval,
dqmHarvestUnit=self.dqmHarvestUnit)
# setting the parameters which need to be set for all the tasks
# sets acquisitionEra, processingVersion, processingString
self.workload.setTaskPropertiesFromWorkload()
self.reportWorkflowToDashboard(self.workload.getDashboardActivity())
return self.workload示例4: __call__
def __call__(self, workloadName, arguments):
"""
_call_
Create a ReDigi workload with the given parameters.
"""
DataProcessing.__call__(self, workloadName, arguments)
# Transform the pileup as required by the CMSSW step
self.pileupConfig = parsePileupConfig(self.mcPileup, self.dataPileup)
# Adjust the pileup splitting
self.procJobSplitArgs.setdefault("deterministicPileup", self.deterministicPileup)
# Adjust the sizePerEvent, timePerEvent and memory for step two and three
if self.stepTwoTimePerEvent is None:
self.stepTwoTimePerEvent = self.timePerEvent
if self.stepTwoSizePerEvent is None:
self.stepTwoSizePerEvent = self.sizePerEvent
if self.stepTwoMemory is None:
self.stepTwoMemory = self.memory
if self.stepThreeTimePerEvent is None:
self.stepThreeTimePerEvent = self.timePerEvent
if self.stepThreeSizePerEvent is None:
self.stepThreeSizePerEvent = self.sizePerEvent
if self.stepThreeMemory is None:
self.stepThreeMemory = self.memory
return self.buildWorkload()示例5: getWorkloadArguments
def getWorkloadArguments():
baseArgs = DataProcessing.getWorkloadArguments()
specArgs = {"RequestType": {"default": "MonteCarloFromGEN", "optional": True,
"attr": "requestType"},
"PrimaryDataset": {"default": None, "type": str,
"optional": True, "validate": primdataset,
"attr": "inputPrimaryDataset", "null": True},
"ConfigCacheUrl": {"default": None, "type": str,
"optional": True, "validate": None,
"attr": "configCacheUrl", "null": False},
"ConfigCacheID": {"default": None, "type": str,
"optional": False, "validate": None,
"attr": "configCacheID", "null": True},
"MCPileup": {"default": None, "type": str,
"optional": True, "validate": dataset,
"attr": "mcPileup", "null": True},
"DataPileup": {"default": None, "type": str,
"optional": True, "validate": dataset,
"attr": "dataPileup", "null": True},
"DeterministicPileup": {"default": False, "type": strToBool,
"optional": True, "validate": None,
"attr": "deterministicPileup", "null": False}}
baseArgs.update(specArgs)
DataProcessing.setDefaultArgumentsProperty(baseArgs)
return baseArgs示例6: __call__
def __call__(self, workloadName, arguments):
"""
_call_
Create a MonteCarloFromGEN workload with the given parameters.
"""
DataProcessing.__call__(self, workloadName, arguments)
return self.buildWorkload()示例7: getWorkloadAssignArgs
def getWorkloadAssignArgs():
baseArgs = DataProcessing.getWorkloadAssignArgs()
specArgs = {
"Override": {"default": {"eos-lfn-prefix": "root://eoscms.cern.ch//eos/cms/store/logs/prod/recent/PromptReco"},
"type": dict},
}
baseArgs.update(specArgs)
DataProcessing.setDefaultArgumentsProperty(baseArgs)
return baseArgs示例8: getWorkloadCreateArgs
def getWorkloadCreateArgs():
baseArgs = DataProcessing.getWorkloadCreateArgs()
specArgs = {"RequestType" : {"default" : "ReReco", "optional" : False},
"TransientOutputModules" : {"default" : [], "type" : makeList,
"attr" : "transientModules", "null" : False}
}
baseArgs.update(specArgs)
DataProcessing.setDefaultArgumentsProperty(baseArgs)
return baseArgs示例9: getWorkloadCreateArgs
def getWorkloadCreateArgs():
baseArgs = DataProcessing.getWorkloadCreateArgs()
specArgs = {"RequestType": {"default": "DQMHarvest", "optional": True},
"ConfigCacheID": {"optional": True, "null": True},
"DQMConfigCacheID": {"optional": False, "attr": "dqmConfigCacheID"},
"DQMUploadUrl": {"optional": False, "attr": "dqmUploadUrl"},
}
baseArgs.update(specArgs)
DataProcessing.setDefaultArgumentsProperty(baseArgs)
return baseArgs示例10: validateSchema
def validateSchema(self, schema):
"""
_validateSchema_
Standard DataProcessing schema validation.
"""
DataProcessing.validateSchema(self, schema)
self.validateConfigCacheExists(configID=schema["DQMConfigCacheID"],
configCacheUrl=schema['ConfigCacheUrl'],
couchDBName=schema["CouchDBName"],
getOutputModules=False)示例11: validateSchema
def validateSchema(self, schema):
"""
_validateSchema_
Standard StdBase schema validation, plus verification
of the ConfigCacheID
"""
DataProcessing.validateSchema(self, schema)
couchUrl = schema.get("ConfigCacheUrl", None) or schema["CouchURL"]
self.validateConfigCacheExists(configID=schema["ConfigCacheID"],
couchURL=couchUrl,
couchDBName=schema["CouchDBName"])
return示例12: __call__
def __call__(self, workloadName, arguments):
"""
_call_
Create a MonteCarloFromGEN workload with the given parameters.
"""
DataProcessing.__call__(self, workloadName, arguments)
# Transform the pileup as required by the CMSSW step
self.pileupConfig = parsePileupConfig(self.mcPileup, self.dataPileup)
# Adjust the pileup splitting
self.procJobSplitArgs.setdefault("deterministicPileup", self.deterministicPileup)
return self.buildWorkload()示例13: validateSchema
def validateSchema(self, schema):
"""
_validateSchema_
Standard StdBase schema validation, plus verification
of the ConfigCacheID
"""
DataProcessing.validateSchema(self, schema)
self.validateConfigCacheExists(configID=schema["ConfigCacheID"],
configCacheUrl=schema['ConfigCacheUrl'],
couchDBName=schema["CouchDBName"],
getOutputModules=False)
return示例14: __init__
def __init__(self):
"""
__init__
Setup parameters that will be later overwritten in the call,
otherwise pylint will complain about them.
"""
DataProcessing.__init__(self)
self.stepTwoMemory = None
self.stepTwoSizePerEvent = None
self.stepTwoTimePerEvent = None
self.stepThreeMemory = None
self.stepThreeSizePerEvent = None
self.stepThreeTimePerEvent = None示例15: getWorkloadArguments
def getWorkloadArguments():
baseArgs = DataProcessing.getWorkloadArguments()
specArgs = {"PrimaryDataset" : {"default" : None, "type" : str,
"optional" : True, "validate" : primdataset,
"attr" : "inputPrimaryDataset", "null" : False},
"ConfigCacheID" : {"default" : None, "type" : str,
"optional" : False, "validate" : None,
"attr" : "configCacheID", "null" : False}}
baseArgs.update(specArgs)
return baseArgs