Python python_compat.izip函数代码示例

def makeEnum(members = None, cls = None, useHash = True):
	members = members or []
	if cls:
		enumID = md5_hex(str(members) + '!' + cls.__name__)[:4]
	else:
		enumID = md5_hex(str(members))[:4]
		cls = type('Enum_%s_%s' % (enumID, str.join('_', members)), (), {})

	def getValue(idx, name):
		if useHash:
			return idx + int(enumID, 16)
		else:
			return idx
	values = lsmap(getValue, enumerate(members))

	cls.enumNames = members
	cls.enumValues = values
	enumMapNV = dict(izip(imap(str.lower, cls.enumNames), cls.enumValues))
	enumMapVN = dict(izip(cls.enumValues, cls.enumNames))
	if len(enumMapNV) != len(enumMapVN):
		raise APIError('Invalid enum definition!')
	def str2enum(cls, value, *args):
		return enumMapNV.get(value.lower(), *args)
	cls.enum2str = enumMapVN.get
	cls.str2enum = classmethod(str2enum)
	for name, value in izip(cls.enumNames, cls.enumValues):
		setattr(cls, name, value)
	return cls

	def _get_just_fun_dict(self, head, align_str):
		just_fun_dict = {'l': str.ljust, 'r': str.rjust, 'c': str.center}
		# just_fun = {id1: str.center, id2: str.rjust, ...}

		def _get_key_format(head_entry, align_str):
			return (head_entry[0], just_fun_dict[align_str])
		return dict(ismap(_get_key_format, izip(head, align_str)))

	def fill_parameter_content(self, pnum, result):
		for (psrc, psrc_len) in izip(self._psrc_list, self._psrc_max_list):
			if psrc_len is not None:
				if pnum < psrc_len:
					psrc.fill_parameter_content(pnum, result)
			else:
				psrc.fill_parameter_content(pnum, result)

	def _get_sandbox_file_list(self, task, sm_list):
		# Prepare all input files
		dep_list = set(ichain(imap(lambda x: x.get_dependency_list(), [task] + sm_list)))
		dep_fn_list = lmap(lambda dep: resolve_path('env.%s.sh' % dep,
			lmap(lambda pkg: get_path_share('', pkg=pkg), os.listdir(get_path_pkg()))), dep_list)
		task_config_dict = dict_union(self._remote_event_handler.get_mon_env_dict(),
			*imap(lambda x: x.get_task_dict(), [task] + sm_list))
		task_config_dict.update({'GC_DEPFILES': str.join(' ', dep_list),
			'GC_USERNAME': self._token.get_user_name(), 'GC_WMS_NAME': self._name})
		task_config_str_list = DictFormat(escape_strings=True).format(
			task_config_dict, format='export %s%s%s\n')
		vn_alias_dict = dict(izip(self._remote_event_handler.get_mon_env_dict().keys(),
			self._remote_event_handler.get_mon_env_dict().keys()))
		vn_alias_dict.update(task.get_var_alias_map())
		vn_alias_str_list = DictFormat(delimeter=' ').format(vn_alias_dict, format='%s%s%s\n')

		# Resolve wildcards in task input files
		def _get_task_fn_list():
			for fpi in task.get_sb_in_fpi_list():
				matched = glob.glob(fpi.path_abs)
				if matched != []:
					for match in matched:
						yield match
				else:
					yield fpi.path_abs
		return lchain([self._remote_event_handler.get_file_list(), dep_fn_list, _get_task_fn_list(), [
			VirtualFile('_config.sh', sorted(task_config_str_list)),
			VirtualFile('_varmap.dat', sorted(vn_alias_str_list))]])

	def discover(self):
		tags = ['h_vmem', 'h_cpu', 's_rt']
		reqs = dict(izip(tags, [WMS.MEMORY, WMS.CPUTIME, WMS.WALLTIME]))
		parser = dict(izip(tags, [int, parseTime, parseTime]))

		proc = LocalProcess(self._configExec, '-sql')
		for queue in imap(str.strip, proc.stdout.iter(timeout = 10)):
			proc_q = LocalProcess(self._configExec, '-sq', queue)
			queueInfo = {'name': queue}
			for line in proc_q.stdout.iter(timeout = 10):
				attr, value = lmap(str.strip, line.split(' ', 1))
				if (attr in tags) and (value != 'INFINITY'):
					queueInfo[reqs[attr]] = parser[attr](value)
			proc_q.status_raise(timeout = 0)
			yield queueInfo
		proc.status_raise(timeout = 0)

	def _statusReturnLineRead(self,line):
		try:
			statusReturnValues = line.split()
			# transform output string to dictionary
			jobinfo = dict(izip(self.statusReturnKeys, statusReturnValues))
			# extract GC and WMS ID, check for consistency
			jobID,wmsID=jobinfo['[email protected]'].split('@')
			if (wmsID != jobinfo['wmsid']):
				raise BackendError("Critical! Unable to match jobs in queue! \n CondorID: %s	Expected: %s \n%s" % ( jobinfo['wmsid'], wmsID, line ))
			jobinfo['jobid']=int(jobID)
			del jobinfo['[email protected]']
			# extract Host and Queue data
			if "@" in jobinfo["RemoteHost"]:
				jobinfo['dest'] = jobinfo["RemoteHost"].split("@")[1] + ': /' + jobinfo.get("Queue","")
			else:
				jobinfo['dest'] = jobinfo["RemoteHost"]
			del jobinfo["RemoteHost"]
			if "Queue" in jobinfo:
				del jobinfo["Queue"]
			# convert status to appropriate format
			status = self._statusMap[jobinfo['status']]
			jobinfo['status'] = self._humanMap[jobinfo['status']]
			return ( jobinfo['jobid'], jobinfo['wmsid'], status, jobinfo )
		except Exception:
			raise BackendError('Error reading job info:\n%s' % line)

	def process(self, dn):
		job_info_dict = None
		try:
			job_info_dict = JobInfoProcessor.process(self, dn)
		except JobResultError:
			logger = logging.getLogger('jobs.results')
			logger.warning('Unable to process job information', exc_info=get_current_exception())
			clear_current_exception()
		if job_info_dict:
			job_data_dict = job_info_dict[JobResult.RAW]
			result = {}

			def get_items_with_key(key_prefix):
				return ifilter(lambda key_value: key_value[0].startswith(key_prefix), job_data_dict.items())

			# parse old job info data format for files
			old_fmt_header = [FileInfo.Hash, FileInfo.NameLocal, FileInfo.NameDest, FileInfo.Path]
			for (file_key, file_data) in get_items_with_key('FILE'):
				file_idx = file_key.replace('FILE', '') or '0'
				result[int(file_idx)] = dict(izip(old_fmt_header, file_data.strip('"').split('  ')))
			# parse new job info data format
			for (file_key, file_data) in get_items_with_key('OUTPUT_FILE'):
				(file_idx, file_prop) = file_key.replace('OUTPUT_FILE_', '').split('_')
				if isinstance(file_data, str):
					file_data = file_data.strip('"')
				file_prop = file_prop.lower().replace('dest', 'namedest').replace('local', 'namelocal')
				result.setdefault(int(file_idx), {})[FileInfo.str2enum(file_prop)] = file_data
			return list(result.values())

	def _acceptLumi(self, block, fi, idxRuns, idxLumi):
		if (idxRuns is None) or (idxLumi is None):
			return True
		fi_meta = fi[DataProvider.Metadata]
		for (run, lumi) in izip(fi_meta[idxRuns], fi_meta[idxLumi]):
			if selectLumi((run, lumi), self._lumi_filter.lookup(block[DataProvider.Nickname], is_selector = False)):
				return True

	def fillParameterInfo(self, pNum, result):
		for (psource, maxN) in izip(self._psourceList, self._psourceMaxList):
			if maxN is not None:
				if pNum < maxN:
					psource.fillParameterInfo(pNum, result)
			else:
				psource.fillParameterInfo(pNum, result)

	def _parseTime(self, time_str):
		result = 0
		entry_map = {'yea': 365 * 24 * 60 * 60, 'day': 24 * 60 * 60, 'hou': 60 * 60, 'min': 60, 'sec': 1}
		tmp = time_str.split()
		for (entry, value) in izip(imap(lambda x: x[:3], tmp[1::2]), imap(int, tmp[::2])):
			result += entry_map[entry] * value
		return result

	def discover(self):
		active = False
		keys = [WMS.MEMORY, WMS.CPUTIME, WMS.WALLTIME]
		parser = dict(izip(keys, [int, parse_time, parse_time]))
		proc = LocalProcess(self._exec, '-q')
		for line in proc.stdout.iter(timeout=10):
			if line.startswith('-'):
				active = True
			elif line.startswith(' '):
				active = False
			elif active:
				fields = lmap(str.strip, line.split()[:4])
				queue_dict = {'name': fields[0]}
				for key, value in ifilter(lambda k_v: not k_v[1].startswith('-'), izip(keys, fields[1:])):
					queue_dict[key] = parser[key](value)
				yield queue_dict
		proc.status_raise(timeout=0)

	def _bound_metric(self, time_step_list, metric_list, lim_low, lim_high):
		time_step_list_truncated = []
		metric_list_truncated = []
		for time_step, metric in izip(time_step_list, metric_list):
			if lim_low < time_step < lim_high:
				metric_list_truncated.append(metric)
				time_step_list_truncated.append(time_step)
		return (time_step_list_truncated, metric_list_truncated)

	def getQueues(self):
		queues = {}
		tags = ['h_vmem', 'h_cpu', 's_rt']
		reqs = dict(izip(tags, [WMS.MEMORY, WMS.CPUTIME, WMS.WALLTIME]))
		parser = dict(izip(tags, [int, parseTime, parseTime]))

		proc = LocalProcess(self._configExec, '-sql')
		for queue in imap(str.strip, proc.stdout.iter(timeout = 10)):
			queues[queue] = dict()
			proc_q = LocalProcess(self._configExec, '-sq %s' % queue)
			for line in proc_q.stdout.iter(timeout = 10):
				attr, value = lmap(str.strip, line.split(' ', 1))
				if (attr in tags) and (value != 'INFINITY'):
					queues[queue][reqs[attr]] = parser[attr](value)
			proc_q.status_raise(timeout = 0)
		proc.status_raise(timeout = 0)
		return queues

	def truncateData(self, timeStep, overAllBandwidth, truncFront, truncBack):
		truncatedTimeStep = []
		truncatedOverAllBandwidth = []
		for currentTimeStep, thisBw in izip(timeStep, overAllBandwidth):
			if (currentTimeStep > truncFront) and (currentTimeStep < truncBack):
				truncatedOverAllBandwidth.append(thisBw)
				truncatedTimeStep.append(currentTimeStep)
		return (truncatedTimeStep, truncatedOverAllBandwidth)

	def _query(self, api, **kwargs):
		key = (self._url, api, tuple(kwargs.items()))
		if key not in SiteDB.queryCache:
			SiteDB.queryCache[key] = self._gjrc.get(api = api, params = kwargs or None)
		data = SiteDB.queryCache[key]
		columns = data['desc']['columns']
		for row in data['result']:
			yield dict(izip(columns, row))