Python DataTask.__init__方法代码示例

# 需要导入模块: from grid_control.tasks.task_data import DataTask [as 别名]
# 或者: from grid_control.tasks.task_data.DataTask import __init__ [as 别名]
	def __init__(self, config, name):
		DataTask.__init__(self, config, name)
		config.set('area files matcher mode', 'ShellStyleMatcher')

		# SCRAM settings
		scram_arch_default = unspecified
		scram_project = config.get_list('scram project', [])
		if scram_project:  # manual scram setup
			if len(scram_project) != 2:
				raise ConfigError('%r needs exactly 2 arguments: <PROJECT> <VERSION>' % 'scram project')
			self._project_area = None
			self._project_area_selector_list = None
			self._scram_project = scram_project[0]
			self._scram_project_version = scram_project[1]
			# ensure project area is not used
			if 'project area' in config.get_option_list():
				raise ConfigError('Cannot specify both %r and %r' % ('scram project', 'project area'))

		else:  # scram setup used from project area
			self._project_area = config.get_dn('project area')
			self._always_matcher = Matcher.create_instance('AlwaysMatcher', config, [''])
			self._project_area_base_fn = config.get_bool('area files basename', True,
				on_change=TriggerInit('sandbox'))
			self._project_area_matcher = config.get_matcher('area files',
				'-.* -config bin lib python module data *.xml *.sql *.db *.cfi *.cff *.py -CVS -work.* *.pcm',
				default_matcher='BlackWhiteMatcher', on_change=TriggerInit('sandbox'))
			self._log.info('Project area found in: %s', self._project_area)

			# try to determine scram settings from environment settings
			scram_path = os.path.join(self._project_area, '.SCRAM')
			scram_env = self._parse_scram_file(os.path.join(scram_path, 'Environment'))
			try:
				self._scram_project = scram_env['SCRAM_PROJECTNAME']
				self._scram_project_version = scram_env['SCRAM_PROJECTVERSION']
			except:
				raise ConfigError('Installed program in project area not recognized.')

			def filter_arch_dir(dn):
				return os.path.isdir(os.path.join(scram_path, dn))
			for arch_dir in sorted(ifilter(filter_arch_dir, os.listdir(scram_path))):
				scram_arch_default = arch_dir

		self._scram_version = config.get('scram version', 'scramv1')
		self._scram_arch = config.get('scram arch', scram_arch_default)

		self._scram_req_list = []
		if config.get_bool('scram arch requirements', True, on_change=None):
			self._scram_req_list.append((WMS.SOFTWARE, 'VO-cms-%s' % self._scram_arch))
		if config.get_bool('scram project requirements', False, on_change=None):
			self._scram_req_list.append((WMS.SOFTWARE, 'VO-cms-%s' % self._scram_project))
		if config.get_bool('scram project version requirements', False, on_change=None):
			self._scram_req_list.append((WMS.SOFTWARE, 'VO-cms-%s' % self._scram_project_version))

# 需要导入模块: from grid_control.tasks.task_data import DataTask [as 别名]
# 或者: from grid_control.tasks.task_data.DataTask import __init__ [as 别名]
	def __init__(self, config, name):
		config.set('se input timeout', '0:30')
		config.set('dataset provider', 'DBS3Provider')
		config.set('dataset splitter', 'EventBoundarySplitter')
		config.set('partition processor', 'CMSPartitionProcessor LocationPartitionProcessor LumiPartitionProcessor')
		config.set('dataset processor', 'LumiDataProcessor', '+=')
		DataTask.__init__(self, config, name)
		self.updateErrorDict(utils.pathShare('gc-run.cmssw.sh', pkg = 'grid_control_cms'))

		# SCRAM settings
		self._configureSCRAMSettings(config)

		self.useReqs = config.getBool('software requirements', True, onChange = None)
		self._projectAreaTarballSE = config.getBool(['se project area', 'se runtime'], True)
		self._projectAreaTarball = config.getWorkPath('cmssw-project-area.tar.gz')

		# Information about search order for software environment
		self.searchLoc = self._getCMSSWPaths(config)
		# Prolog / Epilog script support - warn about old syntax
		self.prolog = TaskExecutableWrapper(config, 'prolog', '')
		self.epilog = TaskExecutableWrapper(config, 'epilog', '')
		if config.getPaths('executable', []) != []:
			raise ConfigError('Prefix executable and argument options with either prolog or epilog!')
		self.arguments = config.get('arguments', '')

		# Get cmssw config files and check their existance
		# Check that for dataset jobs the necessary placeholders are in the config file
		if self.dataSplitter is None:
			self.eventsPerJob = config.get('events per job', '0')
		fragment = config.getPath('instrumentation fragment', utils.pathShare('fragmentForCMSSW.py', pkg = 'grid_control_cms'))
		self.configFiles = self._processConfigFiles(config, list(self._getConfigFiles(config)), fragment,
			autoPrepare = config.getBool('instrumentation', True),
			mustPrepare = (self.dataSplitter is not None))

		# Create project area tarball
		if not os.path.exists(self._projectAreaTarball):
			config.setState(True, 'init', detail = 'sandbox')
		if config.getState('init', detail = 'sandbox'):
			if os.path.exists(self._projectAreaTarball):
				if not utils.getUserBool('CMSSW tarball already exists! Do you want to regenerate it?', True):
					return
			# Generate CMSSW tarball
			if self.projectArea:
				utils.genTarball(self._projectAreaTarball, utils.matchFiles(self.projectArea, self.pattern))
			if self._projectAreaTarballSE:
				config.setState(True, 'init', detail = 'storage')

# 需要导入模块: from grid_control.tasks.task_data import DataTask [as 别名]
# 或者: from grid_control.tasks.task_data.DataTask import __init__ [as 别名]
	def __init__(self, config, name):
		DataTask.__init__(self, config, name)

		# SCRAM settings
		scramArchDefault = noDefault
		scramProject = config.getList('scram project', [])
		if scramProject: # manual scram setup
			if len(scramProject) != 2:
				raise ConfigError('%r needs exactly 2 arguments: <PROJECT> <VERSION>' % 'scram project')
			self._projectArea = None
			self._projectAreaPattern = None
			self._scramProject = scramProject[0]
			self._scramProjectVersion = scramProject[1]
			# ensure project area is not used
			if 'project area' in config.getOptions():
				raise ConfigError('Cannot specify both %r and %r' % ('scram project', 'project area'))

		else: # scram setup used from project area
			self._projectArea = config.getPath('project area')
			self._projectAreaPattern = config.getList('area files', ['-.*', '-config', 'bin', 'lib', 'python', 'module',
				'*/data', '*.xml', '*.sql', '*.db', '*.cf[if]', '*.py', '-*/.git', '-*/.svn', '-*/CVS', '-*/work.*'])
			logging.getLogger('user').info('Project area found in: %s', self._projectArea)

			# try to determine scram settings from environment settings
			scramPath = os.path.join(self._projectArea, '.SCRAM')
			scramEnv = self._parse_scram_file(os.path.join(scramPath, 'Environment'))
			try:
				self._scramProject = scramEnv['SCRAM_PROJECTNAME']
				self._scramProjectVersion = scramEnv['SCRAM_PROJECTVERSION']
			except:
				raise ConfigError('Installed program in project area not recognized.')

			for arch_dir in sorted(ifilter(lambda dn: os.path.isdir(os.path.join(scramPath, dn)), os.listdir(scramPath))):
				scramArchDefault = arch_dir

		self._scramVersion = config.get('scram version', 'scramv1')
		self._scramArch = config.get('scram arch', scramArchDefault)

		self._scramReqs = []
		if config.getBool('scram arch requirements', True, onChange = None):
			self._scramReqs.append((WMS.SOFTWARE, 'VO-cms-%s' % self._scramArch))
		if config.getBool('scram project requirements', False, onChange = None):
			self._scramReqs.append((WMS.SOFTWARE, 'VO-cms-%s' % self._scramProject))
		if config.getBool('scram project version requirements', False, onChange = None):
			self._scramReqs.append((WMS.SOFTWARE, 'VO-cms-%s' % self._scramProjectVersion))

# 需要导入模块: from grid_control.tasks.task_data import DataTask [as 别名]
# 或者: from grid_control.tasks.task_data.DataTask import __init__ [as 别名]
	def __init__(self, config, name):
		DataTask.__init__(self, config, name)
		self._exe = TaskExecutableWrapper(config)

# 需要导入模块: from grid_control.tasks.task_data import DataTask [as 别名]
# 或者: from grid_control.tasks.task_data.DataTask import __init__ [as 别名]
	def __init__(self, config, name):
		config.set('se input timeout', '0:30', override = False)
		config.set('dataset provider', 'DBS3Provider', override = False)
		config.set('dataset splitter', 'EventBoundarySplitter', override = False)
		DataTask.__init__(self, config, name)
		self.errorDict.update(dict(self.updateErrorDict(utils.pathShare('gc-run.cmssw.sh', pkg = 'grid_control_cms'))))

		# SCRAM info
		scramProject = config.getList('scram project', [])
		if len(scramProject):
			self.projectArea = config.getPath('project area', '')
			if len(self.projectArea):
				raise ConfigError('Cannot specify both SCRAM project and project area')
			if len(scramProject) != 2:
				raise ConfigError('SCRAM project needs exactly 2 arguments: PROJECT VERSION')
		else:
			self.projectArea = config.getPath('project area')

		# This works in tandem with provider_dbsv2.py !
		self.selectedLumis = parseLumiFilter(config.get('lumi filter', ''))

		self.useReqs = config.getBool('software requirements', True, onChange = None)
		self.seRuntime = config.getBool('se runtime', False)
		self.runtimePath = config.getWorkPath('runtime.tar.gz')

		if len(self.projectArea):
			defaultPattern = '-.* -config bin lib python module */data *.xml *.sql *.cf[if] *.py -*/.git -*/.svn -*/CVS -*/work.*'
			self.pattern = config.getList('area files', defaultPattern.split())

			if os.path.exists(self.projectArea):
				utils.vprint('Project area found in: %s' % self.projectArea, -1)
			else:
				raise ConfigError('Specified config area %r does not exist!' % self.projectArea)

			scramPath = os.path.join(self.projectArea, '.SCRAM')
			# try to open it
			try:
				fp = open(os.path.join(scramPath, 'Environment'), 'r')
				self.scramEnv = utils.DictFormat().parse(fp, keyParser = {None: str})
			except:
				raise ConfigError('Project area file %s/.SCRAM/Environment cannot be parsed!' % self.projectArea)

			for key in ['SCRAM_PROJECTNAME', 'SCRAM_PROJECTVERSION']:
				if key not in self.scramEnv:
					raise ConfigError('Installed program in project area not recognized.')

			archs = filter(lambda x: os.path.isdir(os.path.join(scramPath, x)) and not x.startswith('.'), os.listdir(scramPath))
			self.scramArch = config.get('scram arch', (archs + [noDefault])[0])
			try:
				fp = open(os.path.join(scramPath, self.scramArch, 'Environment'), 'r')
				self.scramEnv.update(utils.DictFormat().parse(fp, keyParser = {None: str}))
			except:
				raise ConfigError('Project area file .SCRAM/%s/Environment cannot be parsed!' % self.scramArch)
		else:
			self.scramEnv = {
				'SCRAM_PROJECTNAME': scramProject[0],
				'SCRAM_PROJECTVERSION': scramProject[1]
			}
			self.scramArch = config.get('scram arch')

		self.scramVersion = config.get('scram version', 'scramv1')
		if self.scramEnv['SCRAM_PROJECTNAME'] != 'CMSSW':
			raise ConfigError('Project area not a valid CMSSW project area.')

		# Information about search order for software environment
		self.searchLoc = []
		if config.getState('sandbox'):
			userPath = config.get('cmssw dir', '')
			if userPath != '':
				self.searchLoc.append(('CMSSW_DIR_USER', userPath))
			if self.scramEnv.get('RELEASETOP', None):
				projPath = os.path.normpath('%s/../../../../' % self.scramEnv['RELEASETOP'])
				self.searchLoc.append(('CMSSW_DIR_PRO', projPath))
		if len(self.searchLoc):
			utils.vprint('Local jobs will try to use the CMSSW software located here:', -1)
			for i, loc in enumerate(self.searchLoc):
				key, value = loc
				utils.vprint(' %i) %s' % (i + 1, value), -1)

		# Prolog / Epilog script support - warn about old syntax
		self.prolog = TaskExecutableWrapper(config, 'prolog', '')
		self.epilog = TaskExecutableWrapper(config, 'epilog', '')
		if config.getPaths('executable', []) != []:
			raise ConfigError('Prefix executable and argument options with either prolog or epilog!')
		self.arguments = config.get('arguments', '')

		# Get cmssw config files and check their existance
		self.configFiles = []
		cfgDefault = QM(self.prolog.isActive() or self.epilog.isActive(), [], noDefault)
		for cfgFile in config.getPaths('config file', cfgDefault, mustExist = False):
			newPath = config.getWorkPath(os.path.basename(cfgFile))
			if not os.path.exists(newPath):
				if not os.path.exists(cfgFile):
					raise ConfigError('Config file %r not found.' % cfgFile)
				shutil.copyfile(cfgFile, newPath)
			self.configFiles.append(newPath)

		# Check that for dataset jobs the necessary placeholders are in the config file
		self.prepare = config.getBool('prepare config', False)
		fragment = config.getPath('instrumentation fragment',
#.........这里部分代码省略.........

# 需要导入模块: from grid_control.tasks.task_data import DataTask [as 别名]
# 或者: from grid_control.tasks.task_data.DataTask import __init__ [as 别名]
	def __init__(self, config, name):
		DataTask.__init__(self, config, name)