Python process_base.LocalProcess类代码示例

def _get_cms_cert(config):
	config = config.change_view(set_sections=['cms', 'access', 'proxy'])
	try:
		access = AccessToken.create_instance('VomsAccessToken', config, 'cms-proxy')
	except Exception:
		if os.environ.get('X509_USER_PROXY'):
			return os.environ['X509_USER_PROXY']
		raise CMSAuthenticationException('Unable to find grid environment')
	can_submit = ignore_exception(Exception, False, access.can_submit, 5 * 60, True)
	if not can_submit:
		logging.getLogger('access.cms').warning('The grid proxy has expired or is invalid!')
		role = config.get_list('new proxy roles', '', on_change=None)
		timeout = config.get_time('new proxy timeout', 10, on_change=None)
		lifetime = config.get_time('new proxy lifetime', 192 * 60, on_change=None)
		# password in variable name removes it from debug log
		password = getpass.getpass('Please enter proxy password: ')
		try:
			proxy_init_exec = resolve_install_path('voms-proxy-init')
			proc = LocalProcess(proxy_init_exec, '--voms', str.join(':', ['cms'] + role),
				'--valid', '%d:%d' % (lifetime / 60, lifetime % 60), logging=False)
			if password:
				proc.stdin.write(password + '\n')
				proc.stdin.close()
			proc.get_output(timeout=timeout)
		except Exception:
			raise CMSAuthenticationException('Unable to create new grid proxy')
		access = AccessToken.create_instance('VomsAccessToken', config, 'cms-proxy')  # new instance
		can_submit = ignore_exception(Exception, False, access.can_submit, 5 * 60, True)
		if not can_submit:
			raise CMSAuthenticationException('Newly created grid proxy is also invalid')
	return access.get_auth_fn_list()[0]

	def _get_version(self, value):
		old_wd = os.getcwd()
		os.chdir(clean_path(value))
		git_proc = LocalProcess('git', 'rev-parse', '--short', 'HEAD')
		version = git_proc.get_output(timeout=10, raise_errors=False)
		os.chdir(old_wd)
		return version.strip() or 'undefined'

	def _parseTickets(self, cached = True):
		# Return cached results if requested
		if cached and self._cache:
			return self._cache
		# Call klist and parse results
		proc = LocalProcess(self._klistExec)
		self._cache = {}
		try:
			for line in proc.stdout.iter(timeout = 10):
				if line.count('@') and (line.count(':') > 1):
					issued_expires, principal = rsplit(line, '  ', 1)
					issued_expires = issued_expires.replace('/', ' ').split()
					assert(len(issued_expires) % 2 == 0)
					issued_str = str.join(' ', issued_expires[:int(len(issued_expires) / 2)])
					expires_str = str.join(' ', issued_expires[int(len(issued_expires) / 2):])
					parseDate = lambda value, format: time.mktime(time.strptime(value, format))
					if expires_str.count(' ') == 3:
						if len(expires_str.split()[2]) == 2:
							expires = parseDate(expires_str, '%m %d %y %H:%M:%S')
						else:
							expires = parseDate(expires_str, '%m %d %Y %H:%M:%S')
					elif expires_str.count(' ') == 2: # year information is missing
						currentYear = int(time.strftime('%Y'))
						expires = parseDate(expires_str + ' %d' % currentYear, '%b %d %H:%M:%S %Y')
						issued = parseDate(issued_str + ' %d' % currentYear, '%b %d %H:%M:%S %Y')
						if expires < issued: # wraparound at new year
							expires = parseDate(expires_str + ' %d' % (currentYear + 1), '%b %d %H:%M:%S %Y')
					self._cache.setdefault('tickets', {})[principal] = expires
				elif line.count(':') == 1:
					key, value = lmap(str.strip, line.split(':', 1))
					self._cache[key.lower()] = value
		except Exception:
			raise AccessTokenError('Unable to parse kerberos ticket information!')
		proc.status_raise(timeout = 0)
		return self._cache

def get_graph_image(graph_dot):
	proc = LocalProcess('twopi', '-Tpng')
	proc.stdin.write(graph_dot)
	proc.stdin.close()
	if proc.status(timeout = 20) is None:
		return 'Unable to render graph!'
	return proc.stdout.read_log() or 'Empty render result!'

	def _submitJob(self, jobNum, module):
		fd, jdl = tempfile.mkstemp('.jdl')
		try:
			jdlData = self.makeJDL(jobNum, module)
			utils.safeWrite(os.fdopen(fd, 'w'), jdlData)
		except Exception:
			utils.removeFiles([jdl])
			raise BackendError('Could not write jdl data to %s.' % jdl)

		try:
			submitArgs = []
			for key_value in utils.filterDict(self._submitParams, vF = lambda v: v).items():
				submitArgs.extend(key_value)
			submitArgs.append(jdl)

			activity = Activity('submitting job %d' % jobNum)
			proc = LocalProcess(self._submitExec, '--nomsg', '--noint', '--logfile', '/dev/stderr', *submitArgs)

			gcID = None
			for line in ifilter(lambda x: x.startswith('http'), imap(str.strip, proc.stdout.iter(timeout = 60))):
				gcID = line
			retCode = proc.status(timeout = 0, terminate = True)

			activity.finish()

			if (retCode != 0) or (gcID is None):
				if self.explainError(proc, retCode):
					pass
				else:
					self._log.log_process(proc, files = {'jdl': SafeFile(jdl).read()})
		finally:
			utils.removeFiles([jdl])
		return (jobNum, utils.QM(gcID, self._createId(gcID), None), {'jdl': str.join('', jdlData)})

	def _scriptThread(self, script, jobNum = None, jobObj = None, allDict = None):
		try:
			tmp = {}
			if jobObj is not None:
				for key, value in jobObj.get_dict().items():
					tmp[key.upper()] = value
			tmp['WORKDIR'] = self._workPath
			tmp.update(self._task.getTaskConfig())
			if jobNum is not None:
				tmp.update(self._task.getJobConfig(jobNum))
			tmp.update(allDict or {})
			env = dict(os.environ)
			for key, value in tmp.items():
				if not key.startswith('GC_'):
					key = 'GC_' + key
				env[key] = str(value)

			script = self._task.substVars('monitoring script', script, jobNum, tmp)
			if not self._silent:
				proc = LocalProcess(*shlex.split(script), **{'environment': env})
				proc_output = proc.get_output(timeout = self._runningMax)
				if proc_output.strip():
					self._log.info(proc_output.strip())
			else:
				os.system(script)
		except Exception:
			self._log.exception('Error while running user script')

	def cancelJobs(self, allIds):
		if len(allIds) == 0:
			raise StopIteration

		waitFlag = False
		for ids in imap(lambda x: allIds[x:x+5], irange(0, len(allIds), 5)):
			# Delete jobs in groups of 5 - with 5 seconds between groups
			if waitFlag and not utils.wait(5):
				break
			waitFlag = True

			jobNumMap = dict(ids)
			jobs = self.writeWMSIds(ids)

			activity = utils.ActivityLog('cancelling jobs')
			proc = LocalProcess(self._cancelExec, '--noint', '--logfile', '/dev/stderr', '-i', jobs)
			retCode = proc.status(timeout = 60, terminate = True)
			del activity

			# select cancelled jobs
			for deletedWMSId in ifilter(lambda x: x.startswith('- '), proc.stdout.iter()):
				deletedWMSId = self._createId(deletedWMSId.strip('- \n'))
				yield (jobNumMap.get(deletedWMSId), deletedWMSId)

			if retCode != 0:
				if self.explainError(proc, retCode):
					pass
				else:
					self._log.log_process(proc, files = {'jobs': utils.safeRead(jobs)})
			utils.removeFiles([jobs])

	def _scriptThread(self, script, jobNum = None, jobObj = None, allDict = None):
		try:
			tmp = {}
			if jobNum is not None:
				tmp.update(self._task.getSubmitInfo(jobNum))
			if jobObj is not None:
				tmp.update(jobObj.getAll())
			tmp['WORKDIR'] = self._workPath
			tmp.update(self._task.getTaskConfig())
			if jobNum is not None:
				tmp.update(self._task.getJobConfig(jobNum))
				tmp.update(self._task.getSubmitInfo(jobNum))
			tmp.update(allDict or {})
			for key, value in tmp.items():
				if not key.startswith('GC_'):
					key = 'GC_' + key
				os.environ[key] = str(value)

			script = self._task.substVars(script, jobNum, tmp)
			if not self._silent:
				proc = LocalProcess(script)
				self._log.info(proc.get_output(timeout = self._runningMax))
			else:
				os.system(script)
		except Exception:
			self._log.exception('Error while running user script!')

	def _script_thread(self, script, task, jobnum=None, job_obj=None, add_dict=None):
		# Get both task and job config / state dicts
		try:
			tmp = {}
			if job_obj is not None:
				for key, value in job_obj.get_dict().items():
					tmp[key.upper()] = value
			tmp['GC_WORKDIR'] = self._path_work
			if jobnum is not None:
				tmp.update(task.get_job_dict(jobnum))
			tmp.update(add_dict or {})
			env = dict(os.environ)
			for key, value in tmp.items():
				if not key.startswith('GC_'):
					key = 'GC_' + key
				env[key] = str(value)

			script = task.substitute_variables('monitoring script', script, jobnum, tmp)
			if not self._silent:
				proc = LocalProcess(*shlex.split(script), **{'env_dict': env})
				proc_output = proc.get_output(timeout=self._script_timeout)
				if proc_output.strip():
					self._log.info(proc_output.strip())
			else:
				os.system(script)
		except Exception:
			self._log.exception('Error while running user script')
			clear_current_exception()

def ping_host(host):
	proc = LocalProcess('ping', '-Uqnc', 1, '-W', 1, host)
	try:
		tmp = proc.get_output(timeout = 1).splitlines()
		assert(tmp[-1].endswith('ms'))
		return float(tmp[-1].split('/')[-2]) / 1000.
	except Exception:
		return None

	def listWMS_all(self):
		result = []
		proc = LocalProcess(self._exeLCGInfoSites, 'wms')
		for line in proc.stdout.iter(timeout = 10):
			result.append(line.strip())
		proc.status_raise(timeout = 0)
		random.shuffle(result)
		return result

	def _list_endpoint_all(self):
		result = []
		proc = LocalProcess(self._lcg_infosites_exec, 'wms')
		for line in proc.stdout.iter(timeout=10):
			result.append(line.strip())
		proc.status_raise(timeout=0)
		random.shuffle(result)
		return result

	def discover(self):
		proc = LocalProcess(self._exec)
		for line in proc.stdout.iter(timeout=self._timeout):
			if not line.startswith(' ') and len(line) > 1:
				node = line.strip()
			if ('state = ' in line) and ('down' not in line) and ('offline' not in line):
				yield {'name': node}
		proc.status_raise(timeout=0)

	def _getJobsOutput(self, ids):
		if len(ids) == 0:
			raise StopIteration

		basePath = os.path.join(self._outputPath, 'tmp')
		try:
			if len(ids) == 1:
				# For single jobs create single subdir
				tmpPath = os.path.join(basePath, md5(ids[0][0]).hexdigest())
			else:
				tmpPath = basePath
			utils.ensureDirExists(tmpPath)
		except Exception:
			raise BackendError('Temporary path "%s" could not be created.' % tmpPath, BackendError)

		jobNumMap = dict(ids)
		jobs = self.writeWMSIds(ids)

		activity = Activity('retrieving %d job outputs' % len(ids))
		proc = LocalProcess(self._outputExec, '--noint', '--logfile', '/dev/stderr', '-i', jobs, '--dir', tmpPath)

		# yield output dirs
		todo = jobNumMap.values()
		currentJobNum = None
		for line in imap(str.strip, proc.stdout.iter(timeout = 60)):
			if line.startswith(tmpPath):
				todo.remove(currentJobNum)
				outputDir = line.strip()
				if os.path.exists(outputDir):
					if 'GC_WC.tar.gz' in os.listdir(outputDir):
						wildcardTar = os.path.join(outputDir, 'GC_WC.tar.gz')
						try:
							tarfile.TarFile.open(wildcardTar, 'r:gz').extractall(outputDir)
							os.unlink(wildcardTar)
						except Exception:
							self._log.error('Can\'t unpack output files contained in %s', wildcardTar)
				yield (currentJobNum, line.strip())
				currentJobNum = None
			else:
				currentJobNum = jobNumMap.get(self._createId(line), currentJobNum)
		retCode = proc.status(timeout = 0, terminate = True)
		activity.finish()

		if retCode != 0:
			if 'Keyboard interrupt raised by user' in proc.stderr.read(timeout = 0):
				utils.removeFiles([jobs, basePath])
				raise StopIteration
			else:
				self._log.log_process(proc, files = {'jobs': SafeFile(jobs).read()})
			self._log.error('Trying to recover from error ...')
			for dirName in os.listdir(basePath):
				yield (None, os.path.join(basePath, dirName))

		# return unretrievable jobs
		for jobNum in todo:
			yield (jobNum, None)

		utils.removeFiles([jobs, basePath])

	def getNodes(self):
		result = []
		proc = LocalProcess(self._nodesExec)
		for line in proc.stdout.iter():
			if not line.startswith(' ') and len(line) > 1:
				node = line.strip()
			if ('state = ' in line) and ('down' not in line) and ('offline' not in line):
				result.append(node)
		proc.status_raise(timeout = 0)
		if len(result) > 0:
			return result