Python python_compat.lfilter函数代码示例

	def _getSectionKey(self, section):
		tmp = section.split()
		assert(len(tmp) > 0)
		(curSection, curNames, curTags) = (tmp[0], [], {})
		for token in tmp[1:]:
			if ':' in token:
				tag_entry = token.split(':')
				assert(len(tag_entry) == 2)
				curTags[tag_entry[0]] = tag_entry[1]
			elif token:
				curNames.append(token)

		def myIndex(src, value):
			try:
				return src.index(value)
			except Exception:
				return None
		idxClass = myIndex(self._cfgClassSections, curSection)
		idxSection = myIndex(self._cfgSections, curSection)
		if (not self._cfgClassSections) and (not self._cfgSections):
			idxSection = 0
		if (idxClass is not None) or (idxSection is not None): # Section is selected by class or manually
			idxNames = tuple(imap(lambda n: myIndex(self._cfgNames, n), curNames))
			if None not in idxNames: # All names in current section are selected
				curTagNames = lfilter(lambda tn: tn in curTags, self._cfgTagsOrder)
				curTagNamesLeft = lfilter(lambda tn: tn not in self._cfgTagsOrder, curTags)
				idxTags = lmap(lambda tn: myIndex(self._cfgTags, (tn, curTags[tn])), curTagNames)
				if (None not in idxTags) and not curTagNamesLeft:
					return (idxClass, idxSection, idxNames, idxTags)

	def _get_section_key(self, section):
		tmp = section.split()
		if not tmp:
			raise ConfigError('Invalid config section %r' % section)
		(cur_section, cur_name_list, cur_tag_map) = (tmp[0], [], {})
		for token in tmp[1:]:
			if ':' in token:
				tag_entry = token.split(':')
				if len(tag_entry) != 2:
					raise ConfigError('Invalid config tag in section %r' % section)
				cur_tag_map[tag_entry[0]] = tag_entry[1]
			elif token:
				cur_name_list.append(token)

		class_section_idx = safe_index(self._class_section_list, cur_section)
		section_idx = safe_index(self._section_list, cur_section)
		if (not self._class_section_list) and (not self._section_list):
			section_idx = 0
		if (class_section_idx is not None) or (section_idx is not None):
			# Section is selected by class or manually
			name_idx_tuple = tuple(imap(lambda n: safe_index(self._section_name_list, n), cur_name_list))
			if None not in name_idx_tuple:  # All names in current section are selected
				cur_tag_name_list = lfilter(cur_tag_map.__contains__, self._section_tag_order)
				left_tag_name_list = lfilter(lambda tn: tn not in self._section_tag_order, cur_tag_map)
				tag_tuple_list = imap(lambda tn: (tn, cur_tag_map[tn]), cur_tag_name_list)
				tag_idx_tuple = tuple(imap(lambda tt: safe_index(self._section_tag_list, tt), tag_tuple_list))
				if (None not in tag_idx_tuple) and not left_tag_name_list:
					return (class_section_idx, section_idx, name_idx_tuple, tag_idx_tuple)

	def makeJDL(self, jobNum, module):
		cfgPath = os.path.join(self._jobPath, 'job_%d.var' % jobNum)
		sbIn = lmap(lambda d_s_t: d_s_t[1], self._getSandboxFilesIn(module))
		sbOut = lmap(lambda d_s_t: d_s_t[2], self._getSandboxFilesOut(module))
		wcList = lfilter(lambda x: '*' in x, sbOut)
		if len(wcList):
			self._writeJobConfig(cfgPath, jobNum, module, {'GC_WC': str.join(' ', wcList)})
			sandboxOutJDL = lfilter(lambda x: x not in wcList, sbOut) + ['GC_WC.tar.gz']
		else:
			self._writeJobConfig(cfgPath, jobNum, module, {})
			sandboxOutJDL = sbOut
		# Warn about too large sandboxes
		sbSizes = lmap(os.path.getsize, sbIn)
		if sbSizes and (self._warnSBSize > 0) and (sum(sbSizes) > self._warnSBSize * 1024 * 1024):
			if not utils.getUserBool('Sandbox is very large (%d bytes) and can cause issues with the WMS! Do you want to continue?' % sum(sbSizes), False):
				sys.exit(os.EX_OK)
			self._warnSBSize = 0

		reqs = self.brokerSite.brokerAdd(module.getRequirements(jobNum), WMS.SITES)
		formatStrList = lambda strList: '{ %s }' % str.join(', ', imap(lambda x: '"%s"' % x, strList))
		contents = {
			'Executable': '"gc-run.sh"',
			'Arguments': '"%d"' % jobNum,
			'StdOutput': '"gc.stdout"',
			'StdError': '"gc.stderr"',
			'InputSandbox': formatStrList(sbIn + [cfgPath]),
			'OutputSandbox': formatStrList(sandboxOutJDL),
			'VirtualOrganisation': '"%s"' % self.vo,
			'Rank': '-other.GlueCEStateEstimatedResponseTime',
			'RetryCount': 2
		}
		return self._jdl_writer.format(reqs, contents)

	def _check_get_jobnum_list(self, task, wms, jobnum_list):
		if self._defect_tries:
			num_defect = len(self._defect_counter)  # Waiting list gets larger in case reported == []
			num_wait = num_defect - max(1, int(num_defect / 2 ** self._defect_raster))
			jobnum_list_wait = self._sample(self._defect_counter, num_wait)
			jobnum_list = lfilter(lambda jobnum: jobnum not in jobnum_list_wait, jobnum_list)

		(change, jobnum_list_timeout, reported) = JobManager._check_get_jobnum_list(
			self, task, wms, jobnum_list)
		for jobnum in reported:
			self._defect_counter.pop(jobnum, None)

		if self._defect_tries and (change is not None):
			# make 'raster' iteratively smaller
			self._defect_raster += 1
			if reported:
				self._defect_raster = 1
			for jobnum in ifilter(lambda x: x not in reported, jobnum_list):
				self._defect_counter[jobnum] = self._defect_counter.get(jobnum, 0) + 1
			jobnum_list_kick = lfilter(lambda jobnum: self._defect_counter[jobnum] >= self._defect_tries,
				self._defect_counter)
			if (len(reported) == 0) and (len(jobnum_list) == 1):
				jobnum_list_kick.extend(jobnum_list)
			for jobnum in set(jobnum_list_kick):
				jobnum_list_timeout.append(jobnum)
				self._defect_counter.pop(jobnum)

		return (change, jobnum_list_timeout, reported)

	def _getSubmissionJobs(self, maxsample):
		# Get list of submittable jobs
		readyList = self.jobDB.getJobs(ClassSelector(JobClass.READY))
		retryOK = readyList
		defaultJob = Job()
		if self._job_retries >= 0:
			retryOK = lfilter(lambda x: self.jobDB.get(x, defaultJob).attempt - 1 < self._job_retries, readyList)
		modOK = lfilter(self._task.canSubmit, readyList)
		jobList = set.intersection(set(retryOK), set(modOK))

		if self._showBlocker and readyList and not jobList: # No submission but ready jobs
			err = []
			err += utils.QM((len(retryOK) > 0) and (len(modOK) == 0), [], ['have hit their maximum number of retries'])
			err += utils.QM((len(retryOK) == 0) and (len(modOK) > 0), [], ['are vetoed by the task module'])
			self._log_user_time.warning('All remaining jobs %s!', str.join(utils.QM(retryOK or modOK, ' or ', ' and '), err))
		self._showBlocker = not (len(readyList) > 0 and len(jobList) == 0)

		# Determine number of jobs to submit
		submit = len(jobList)
		if self._njobs_inqueue > 0:
			submit = min(submit, self._njobs_inqueue - self.jobDB.getJobsN(ClassSelector(JobClass.ATWMS)))
		if self._njobs_inflight > 0:
			submit = min(submit, self._njobs_inflight - self.jobDB.getJobsN(ClassSelector(JobClass.PROCESSING)))
		if self._continuous and (maxsample > 0):
			submit = min(submit, maxsample)
		submit = max(submit, 0)

		if self._do_shuffle:
			return self._sample(jobList, submit)
		return sorted(jobList)[:submit]

    def cms_name_to_se(self, cms_name):
        cms_name = cms_name.replace('*', '.*')
        cms_name = cms_name.replace('%', '.*')
        cms_name_regex = re.compile(cms_name)

        psn_site_names = lfilter(lambda site: site['type'] == 'psn' and cms_name_regex.match(site['alias']), self._site_names())
        site_names = set(imap(lambda x: x['site_name'], psn_site_names))
        site_resources = lfilter(lambda x: x['site_name'] in site_names, self._site_resources())
        host_list = lfilter(lambda x: x['type'] == 'SE', site_resources)
        host_list = lmap(lambda x: x['fqdn'], host_list)
        return host_list

def list_parameters(opts, psource):
	(result, needGCParam) = get_parameters(opts, psource)
	enabledOutput = opts.output.split(',')
	output = lfilter(lambda k: not opts.output or k in enabledOutput, psource.getJobKeys())
	stored = lfilter(lambda k: k.untracked == False, output)
	untracked = lfilter(lambda k: k.untracked == True, output)

	if opts.collapse > 0:
		result_old = result
		result = {}
		result_nicks = {}
		head = [('COLLATE_JOBS', '# of jobs')]
		if 'DATASETSPLIT' in stored:
			stored.remove('DATASETSPLIT')
			if opts.collapse == 1:
				stored.append('DATASETNICK')
				head.append(('DATASETNICK', 'DATASETNICK'))
			elif opts.collapse == 2:
				head.append(('COLLATE_NICK', '# of nicks'))
		for pset in result_old:
			if ('DATASETSPLIT' in pset) and (opts.collapse == 1):
				pset.pop('DATASETSPLIT')
			nickname = None
			if ('DATASETNICK' in pset) and (opts.collapse == 2):
				nickname = pset.pop('DATASETNICK')
			h = md5_hex(repr(lmap(pset.get, stored)))
			result.setdefault(h, []).append(pset)
			result_nicks.setdefault(h, set()).add(nickname)

		def doCollate(h):
			tmp = result[h][0]
			tmp['COLLATE_JOBS'] = len(result[h])
			tmp['COLLATE_NICK'] = len(result_nicks[h])
			return tmp
		result = lmap(doCollate, result)
	else:
		head = [('GC_JOB_ID', '#')]
		if needGCParam:
			head.append(('GC_PARAM', 'GC_PARAM'))
	if opts.active:
		head.append((ParameterInfo.ACTIVE, 'ACTIVE'))
	if opts.visible:
		stored = opts.visible.split(',')
	head.extend(sorted(izip(stored, stored)))
	if opts.untracked:
		head.extend(sorted(imap(lambda n: (n, '(%s)' % n), ifilter(lambda n: n not in ['GC_PARAM', 'GC_JOB_ID'], untracked))))
	utils.vprint('')
	utils.printTabular(head, result)

	def _get_possible_merge_categories(self, map_cat2desc):
		# Get dictionary with categories that will get merged when removing a variable
		def _eq_dict(dict_a, dict_b, key):
			# Merge parameters to reach category goal - NP hard problem, so be greedy and quick!
			dict_a = dict(dict_a)
			dict_b = dict(dict_b)
			dict_a.pop(key)
			dict_b.pop(key)
			return dict_a == dict_b

		var_key_result = {}
		cat_key_search_dict = {}
		for cat_key in map_cat2desc:
			for var_key in map_cat2desc[cat_key]:
				if var_key not in cat_key_search_dict:
					cat_key_search = set(map_cat2desc.keys())
				else:
					cat_key_search = cat_key_search_dict[var_key]
				if cat_key_search:
					matches = lfilter(lambda ck: _eq_dict(map_cat2desc[cat_key],
						map_cat2desc[ck], var_key), cat_key_search)
					if matches:
						cat_key_search_dict[var_key] = cat_key_search.difference(set(matches))
						var_key_result.setdefault(var_key, []).append(matches)
		return var_key_result

	def _processCfg(self, tar, cfg):
		cfgSummary = {}
		cfgContent = bytes2str(tar.extractfile('%s/config' % cfg).read())
		cfgHashResult = bytes2str(tar.extractfile('%s/hash' % cfg).read()).splitlines()
		cfgHash = cfgHashResult[-1].strip()
		cfgSummary = {'CMSSW_CONFIG_FILE': cfg, 'CMSSW_CONFIG_HASH': cfgHash}
		cfgSummary['CMSSW_CONFIG_CONTENT'] = self._cfgStore.setdefault(cfgSummary[self._mergeKey], cfgContent)
		# Read global tag from config file - first from hash file, then from config file
		if cfgHash not in self._gtStore:
			gtLines = lfilter(lambda x: x.startswith('globaltag:'), cfgHashResult)
			if gtLines:
				self._gtStore[cfgHash] = gtLines[-1].split(':')[1].strip()
		if cfgHash not in self._gtStore:
			try:
				cfgContentEnv = utils.execWrapper(cfgContent)
				self._gtStore[cfgHash] = cfgContentEnv['process'].GlobalTag.globaltag.value()
			except Exception:
				self._gtStore[cfgHash] = 'unknown:All'
		cfgSummary['CMSSW_GLOBALTAG'] = self._gtStore[cfgHash]
		# Get annotation from config content
		def searchConfigFile(key, regex, default):
			try:
				tmp = re.compile(regex).search(cfgContent.group(1).strip('\"\' '))
			except Exception:
				tmp = None
			if tmp:
				cfgSummary[key] = tmp
			else:
				cfgSummary[key] = default
		searchConfigFile('CMSSW_ANNOTATION', r'.*annotation.*=.*cms.untracked.string.*\((.*)\)', None)
		searchConfigFile('CMSSW_DATATIER', r'.*dataTier.*=.*cms.untracked.string.*\((.*)\)', 'USER')
		cfgReport = xml.dom.minidom.parseString(bytes2str(tar.extractfile('%s/report.xml' % cfg).read()))
		evRead = sum(imap(lambda x: int(readTag(x, 'EventsRead')), cfgReport.getElementsByTagName('InputFile')))
		return (cfgSummary, cfgReport, evRead)

	def _broker(self, reqs, items):
		if not self._item_list_discovered:
			return FilterBroker._broker(self, reqs, self._item_list_start)  # Use user constrained items

		# Match items which fulfill the requirements
		def _matcher(props):
			for key, value in reqs:
				if props.get(key) is None:
					continue
				if value >= props[key]:
					return False
			return True
		# Apply sort order and give matching entries as preselection to FilterBroker
		items = lfilter(lambda x: _matcher(self._item_list_discovered[x]),
			self._item_list_start or self._item_list_sorted)
		return FilterBroker._broker(self, reqs, lfilter(items.__contains__, self._item_list_sorted))

	def _getCategoryStateSummary(self):
		(catStateDict, catDescDict, catSubcatDict) = CategoryBaseReport._getCategoryStateSummary(self)
		# Used for quick calculations
		catLenDict = {}
		for catKey in catStateDict:
			catLenDict[catKey] = sum(catStateDict[catKey].values())
		# Merge successfully completed categories
		self._mergeCats(catStateDict, catDescDict, catSubcatDict, catLenDict,
			'Completed subtasks', lfilter(lambda catKey:
				(len(catStateDict[catKey]) == 1) and (Job.SUCCESS in catStateDict[catKey]), catStateDict))
		# Next merge steps shouldn't see non-dict catKeys in catDescDict
		hiddenDesc = {}
		for catKey in ifilter(lambda catKey: not isinstance(catDescDict[catKey], dict), list(catDescDict)):
			hiddenDesc[catKey] = catDescDict.pop(catKey)
		# Merge categories till goal is reached
		self._mergeCatsWithGoal(catStateDict, catDescDict, catSubcatDict, catLenDict, hiddenDesc)
		# Remove redundant variables from description
		varKeyResult = self._getKeyMergeResults(catDescDict)
		self._clearCategoryDesc(varKeyResult, catDescDict)
		# Restore hidden descriptions
		catDescDict.update(hiddenDesc)
		# Enforce category maximum - merge categories with the least amount of jobs
		if len(catStateDict) != self._catMax:
			self._mergeCats(catStateDict, catDescDict, catSubcatDict, catLenDict, 'Remaining subtasks',
				sorted(catStateDict, key = lambda catKey: -catLenDict[catKey])[self._catMax - 1:])
		# Finalize descriptions:
		if len(catDescDict) == 1:
			catDescDict[list(catDescDict.keys())[0]] = 'All jobs'
		return (catStateDict, catDescDict, catSubcatDict)

	def _fillContentWithIncludes(self, configFile, searchPaths, configContent):
		log = logging.getLogger(('config.%s' % utils.getRootName(configFile)).rstrip('.').lower())
		log.log(logging.INFO1, 'Reading config file %s', configFile)
		configFile = utils.resolvePath(configFile, searchPaths, ErrorClass = ConfigError)
		configFileLines = SafeFile(configFile).readlines()

		# Single pass, non-recursive list retrieval
		tmpConfigContent = {}
		self._fillContentSingleFile(configFile, configFileLines, searchPaths, tmpConfigContent)
		def getFlatList(section, option):
			for (opt, value, src) in tmpConfigContent.get(section, []):
				try:
					if opt == option:
						for entry in parseList(value, None):
							yield entry
				except Exception:
					raise ConfigError('Unable to parse [%s] %s from %s' % (section, option, src))

		newSearchPaths = [os.path.dirname(configFile)]
		# Add entries from include statement recursively
		for includeFile in getFlatList('global', 'include'):
			self._fillContentWithIncludes(includeFile, searchPaths + newSearchPaths, configContent)
		# Process all other entries in current file
		self._fillContentSingleFile(configFile, configFileLines, searchPaths, configContent)
		# Override entries in current config file
		for overrideFile in getFlatList('global', 'include override'):
			self._fillContentWithIncludes(overrideFile, searchPaths + newSearchPaths, configContent)
		# Filter special global options
		if configContent.get('global', []):
			configContent['global'] = lfilter(lambda opt_v_s: opt_v_s[0] not in ['include', 'include override'], configContent['global'])
		return searchPaths + newSearchPaths

	def format(self, printSection = False, printDefault = False, default = noDefault, source = '', wraplen = 33):
		if (self.value == noDefault) or (not printDefault and (self.value == default)):
			return ''
		if printSection:
			prefix = '[%s] %s' % (self.section, self.option)
		else:
			prefix = self.option
		prefix += ' %s' % self.opttype

		line_list = lfilter(lambda x: x != '', imap(str.strip, self.value.strip().splitlines()))
		if not line_list:
			line_list = [prefix] # just prefix - without trailing whitespace
		elif len(line_list) > 1:
			line_list = [prefix] + line_list # prefix on first line - rest on other lines
		else:
			line_list = [prefix + ' ' + line_list[0]] # everything on one line

		result = ''
		for line in line_list:
			if not result: # first line:
				if source and (len(line) >= wraplen):
					result += '; source: ' + source + '\n'
				elif source:
					result = line.ljust(wraplen) + '  ; ' + source + '\n'
					continue
			else:
				result += '\t'
			result += line + '\n'
		return result.rstrip()

	def __new__(cls, *psrc_list):
		def _select_extensive_psrc(psrc):
			return (not isinstance(psrc, int)) and (psrc.get_parameter_len() is not None)
		psrc_list = _strip_null_sources(psrc_list)
		if len(lfilter(_select_extensive_psrc, psrc_list)) < 2:
			return ZipLongParameterSource(*psrc_list)
		return MultiParameterSource.__new__(cls, *psrc_list)

	def processBlock(self, block):
		if self._emptyFiles:
			block[DataProvider.FileList] = lfilter(lambda fi: fi[DataProvider.NEntries] != 0, block[DataProvider.FileList])
		if self._emptyBlock:
			if (block[DataProvider.NEntries] == 0) or not block[DataProvider.FileList]:
				return
		return block