本文整理汇总了Python中grid_control.tasks.task_data.DataTask类的典型用法代码示例。如果您正苦于以下问题:Python DataTask类的具体用法?Python DataTask怎么用?Python DataTask使用的例子?那么恭喜您, 这里精选的类代码示例或许可以为您提供帮助。
在下文中一共展示了DataTask类的15个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: __init__
def __init__(self, config, name):
DataTask.__init__(self, config, name)
config.set('area files matcher mode', 'ShellStyleMatcher')
# SCRAM settings
scram_arch_default = unspecified
scram_project = config.get_list('scram project', [])
if scram_project: # manual scram setup
if len(scram_project) != 2:
raise ConfigError('%r needs exactly 2 arguments: <PROJECT> <VERSION>' % 'scram project')
self._project_area = None
self._project_area_selector_list = None
self._scram_project = scram_project[0]
self._scram_project_version = scram_project[1]
# ensure project area is not used
if 'project area' in config.get_option_list():
raise ConfigError('Cannot specify both %r and %r' % ('scram project', 'project area'))
else: # scram setup used from project area
self._project_area = config.get_dn('project area')
self._always_matcher = Matcher.create_instance('AlwaysMatcher', config, [''])
self._project_area_base_fn = config.get_bool('area files basename', True,
on_change=TriggerInit('sandbox'))
self._project_area_matcher = config.get_matcher('area files',
'-.* -config bin lib python module data *.xml *.sql *.db *.cfi *.cff *.py -CVS -work.* *.pcm',
default_matcher='BlackWhiteMatcher', on_change=TriggerInit('sandbox'))
self._log.info('Project area found in: %s', self._project_area)
# try to determine scram settings from environment settings
scram_path = os.path.join(self._project_area, '.SCRAM')
scram_env = self._parse_scram_file(os.path.join(scram_path, 'Environment'))
try:
self._scram_project = scram_env['SCRAM_PROJECTNAME']
self._scram_project_version = scram_env['SCRAM_PROJECTVERSION']
except:
raise ConfigError('Installed program in project area not recognized.')
def filter_arch_dir(dn):
return os.path.isdir(os.path.join(scram_path, dn))
for arch_dir in sorted(ifilter(filter_arch_dir, os.listdir(scram_path))):
scram_arch_default = arch_dir
self._scram_version = config.get('scram version', 'scramv1')
self._scram_arch = config.get('scram arch', scram_arch_default)
self._scram_req_list = []
if config.get_bool('scram arch requirements', True, on_change=None):
self._scram_req_list.append((WMS.SOFTWARE, 'VO-cms-%s' % self._scram_arch))
if config.get_bool('scram project requirements', False, on_change=None):
self._scram_req_list.append((WMS.SOFTWARE, 'VO-cms-%s' % self._scram_project))
if config.get_bool('scram project version requirements', False, on_change=None):
self._scram_req_list.append((WMS.SOFTWARE, 'VO-cms-%s' % self._scram_project_version))
示例2: getTaskConfig
def getTaskConfig(self):
data = DataTask.getTaskConfig(self)
data['SCRAM_VERSION'] = self._scramVersion
data['SCRAM_ARCH'] = self._scramArch
data['SCRAM_PROJECTNAME'] = self._scramProject
data['SCRAM_PROJECTVERSION'] = self._scramProjectVersion
return data
示例3: get_job_dict
def get_job_dict(self, jobnum):
job_env_dict = DataTask.get_job_dict(self, jobnum)
job_env_dict['SCRAM_VERSION'] = self._scram_version
job_env_dict['SCRAM_ARCH'] = self._scram_arch
job_env_dict['SCRAM_PROJECTNAME'] = self._scram_project
job_env_dict['SCRAM_PROJECTVERSION'] = self._scram_project_version
return job_env_dict
示例4: getSBInFiles
def getSBInFiles(self):
files = DataTask.getSBInFiles(self) + self.prolog.getSBInFiles() + self.epilog.getSBInFiles()
for cfgFile in self.configFiles:
files.append(utils.Result(pathAbs = cfgFile, pathRel = os.path.basename(cfgFile)))
if len(self.projectArea) and not self._projectAreaTarballSE:
files.append(utils.Result(pathAbs = self._projectAreaTarball, pathRel = os.path.basename(self._projectAreaTarball)))
return files + [utils.Result(pathAbs = utils.pathShare('gc-run.cmssw.sh', pkg = 'grid_control_cms'), pathRel = 'gc-run.cmssw.sh')]
示例5: getVarNames
def getVarNames(self):
result = DataTask.getVarNames(self)
if self.dataSplitter == None:
result.append('MAX_EVENTS')
if self.selectedLumis:
result.append('LUMI_RANGE')
return result
示例6: getJobConfig
def getJobConfig(self, jobNum):
data = DataTask.getJobConfig(self, jobNum)
if self.dataSplitter == None:
data['MAX_EVENTS'] = self.eventsPerJob
if self.selectedLumis:
data['LUMI_RANGE'] = self.getActiveLumiFilter(self.selectedLumis)
return data
示例7: __init__
def __init__(self, config, name):
config.set('se input timeout', '0:30')
config.set('dataset provider', 'DBS3Provider')
config.set('dataset splitter', 'EventBoundarySplitter')
config.set('partition processor', 'CMSPartitionProcessor LocationPartitionProcessor LumiPartitionProcessor')
config.set('dataset processor', 'LumiDataProcessor', '+=')
DataTask.__init__(self, config, name)
self.updateErrorDict(utils.pathShare('gc-run.cmssw.sh', pkg = 'grid_control_cms'))
# SCRAM settings
self._configureSCRAMSettings(config)
self.useReqs = config.getBool('software requirements', True, onChange = None)
self._projectAreaTarballSE = config.getBool(['se project area', 'se runtime'], True)
self._projectAreaTarball = config.getWorkPath('cmssw-project-area.tar.gz')
# Information about search order for software environment
self.searchLoc = self._getCMSSWPaths(config)
# Prolog / Epilog script support - warn about old syntax
self.prolog = TaskExecutableWrapper(config, 'prolog', '')
self.epilog = TaskExecutableWrapper(config, 'epilog', '')
if config.getPaths('executable', []) != []:
raise ConfigError('Prefix executable and argument options with either prolog or epilog!')
self.arguments = config.get('arguments', '')
# Get cmssw config files and check their existance
# Check that for dataset jobs the necessary placeholders are in the config file
if self.dataSplitter is None:
self.eventsPerJob = config.get('events per job', '0')
fragment = config.getPath('instrumentation fragment', utils.pathShare('fragmentForCMSSW.py', pkg = 'grid_control_cms'))
self.configFiles = self._processConfigFiles(config, list(self._getConfigFiles(config)), fragment,
autoPrepare = config.getBool('instrumentation', True),
mustPrepare = (self.dataSplitter is not None))
# Create project area tarball
if not os.path.exists(self._projectAreaTarball):
config.setState(True, 'init', detail = 'sandbox')
if config.getState('init', detail = 'sandbox'):
if os.path.exists(self._projectAreaTarball):
if not utils.getUserBool('CMSSW tarball already exists! Do you want to regenerate it?', True):
return
# Generate CMSSW tarball
if self.projectArea:
utils.genTarball(self._projectAreaTarball, utils.matchFiles(self.projectArea, self.pattern))
if self._projectAreaTarballSE:
config.setState(True, 'init', detail = 'storage')
示例8: __init__
def __init__(self, config, name):
DataTask.__init__(self, config, name)
# SCRAM settings
scramArchDefault = noDefault
scramProject = config.getList('scram project', [])
if scramProject: # manual scram setup
if len(scramProject) != 2:
raise ConfigError('%r needs exactly 2 arguments: <PROJECT> <VERSION>' % 'scram project')
self._projectArea = None
self._projectAreaPattern = None
self._scramProject = scramProject[0]
self._scramProjectVersion = scramProject[1]
# ensure project area is not used
if 'project area' in config.getOptions():
raise ConfigError('Cannot specify both %r and %r' % ('scram project', 'project area'))
else: # scram setup used from project area
self._projectArea = config.getPath('project area')
self._projectAreaPattern = config.getList('area files', ['-.*', '-config', 'bin', 'lib', 'python', 'module',
'*/data', '*.xml', '*.sql', '*.db', '*.cf[if]', '*.py', '-*/.git', '-*/.svn', '-*/CVS', '-*/work.*'])
logging.getLogger('user').info('Project area found in: %s', self._projectArea)
# try to determine scram settings from environment settings
scramPath = os.path.join(self._projectArea, '.SCRAM')
scramEnv = self._parse_scram_file(os.path.join(scramPath, 'Environment'))
try:
self._scramProject = scramEnv['SCRAM_PROJECTNAME']
self._scramProjectVersion = scramEnv['SCRAM_PROJECTVERSION']
except:
raise ConfigError('Installed program in project area not recognized.')
for arch_dir in sorted(ifilter(lambda dn: os.path.isdir(os.path.join(scramPath, dn)), os.listdir(scramPath))):
scramArchDefault = arch_dir
self._scramVersion = config.get('scram version', 'scramv1')
self._scramArch = config.get('scram arch', scramArchDefault)
self._scramReqs = []
if config.getBool('scram arch requirements', True, onChange = None):
self._scramReqs.append((WMS.SOFTWARE, 'VO-cms-%s' % self._scramArch))
if config.getBool('scram project requirements', False, onChange = None):
self._scramReqs.append((WMS.SOFTWARE, 'VO-cms-%s' % self._scramProject))
if config.getBool('scram project version requirements', False, onChange = None):
self._scramReqs.append((WMS.SOFTWARE, 'VO-cms-%s' % self._scramProjectVersion))
示例9: getTaskConfig
def getTaskConfig(self):
data = DataTask.getTaskConfig(self)
data.update(dict(self.searchLoc))
data['CMSSW_OLD_RELEASETOP'] = self.scramEnv.get('RELEASETOP', None)
data['SCRAM_ARCH'] = self.scramArch
data['SCRAM_VERSION'] = self.scramVersion
data['SCRAM_PROJECTVERSION'] = self.scramEnv['SCRAM_PROJECTVERSION']
data['GZIP_OUT'] = utils.QM(self.gzipOut, 'yes', 'no')
data['SE_RUNTIME'] = utils.QM(self._projectAreaTarballSE, 'yes', 'no')
data['HAS_RUNTIME'] = utils.QM(len(self.projectArea), 'yes', 'no')
data['CMSSW_CONFIG'] = str.join(' ', imap(os.path.basename, self.configFiles))
if self.prolog.isActive():
data['CMSSW_PROLOG_EXEC'] = self.prolog.getCommand()
data['CMSSW_PROLOG_SB_IN_FILES'] = str.join(' ', imap(lambda x: x.pathRel, self.prolog.getSBInFiles()))
data['CMSSW_PROLOG_ARGS'] = self.prolog.getArguments()
if self.epilog.isActive():
data['CMSSW_EPILOG_EXEC'] = self.epilog.getCommand()
data['CMSSW_EPILOG_SB_IN_FILES'] = str.join(' ', imap(lambda x: x.pathRel, self.epilog.getSBInFiles()))
data['CMSSW_EPILOG_ARGS'] = self.epilog.getArguments()
return data
示例10: get_requirement_list
def get_requirement_list(self, jobnum):
# Get job requirements
return DataTask.get_requirement_list(self, jobnum) + self._scram_req_list
示例11: getJobArguments
def getJobArguments(self, jobNum):
return DataTask.getJobArguments(self, jobNum) + ' ' + self._exeWrap.getArguments()
示例12: get_dependency_list
def get_dependency_list(self):
return DataTask.get_dependency_list(self) + ['cmssw']
示例13: getVarNames
def getVarNames(self):
result = DataTask.getVarNames(self)
if self.dataSplitter is None:
result.append('MAX_EVENTS')
return result
示例14: get_sb_in_fpi_list
def get_sb_in_fpi_list(self):
return DataTask.get_sb_in_fpi_list(self) + self._exe.get_sb_in_fpi_list()
示例15: getSEInFiles
def getSEInFiles(self):
files = DataTask.getSEInFiles(self)
if len(self.projectArea) and self.seRuntime:
return files + [('CMSSW runtime', self.runtimePath, self.taskID + '.tar.gz')]
return files