Python Utility.Config类代码示例

    def __init__(self):
        logger.info("crabbackend init")
        super(CRABBackend, self).__init__()
        config = Config.getConfig('CMSSW')
        shell = Shell(os.path.join(config['CMSSW_SETUP'], 'CMSSW_generic.sh')) 
        #shell = Shell(os.path.join(config['CMSSW_SETUP'], 'CMSSW_generic.sh'),
        #              [config['CMSSW_VERSION'], config['CRAB_VERSION']])
        self.crab_env = shell.env

        config = Config.getConfig('CRAB_CFG')
        self.server_name = config['server_name']
        self.apiresource = config['apiresource']
        self.userproxy = config['userproxy']
        self.asyncdest = config['asyncdest']
        logger.info("asyncdest %s" % self.asyncdest )

	def run(self):
		# create connection
		self.log.info('Connecting to LGI project %s server %s'%(self.res._project, self.res._url))
		self.res.connect()
		self.queued = None
		config = Config.getConfig('LGI')

		# LGI update loop
		self.log.debug('Starting LGIResourceThread main loop')
		while not self.should_stop():
			now = time.time()

			try:
				work = [self._workForApp(app) for app in self.res.getApplications()]
				totalwork = sum(work)
				if self.log.isEnabledFor('DEBUG'):
					self.log.debug('LGI pending work: %s'%(dict(zip([str(x) for x in self.res.getApplications()], work))))
				if self.queued != totalwork:
					self.log.info('LGI jobs: %d waiting'%totalwork)
				self.queued = totalwork
			except Exception, e:
				self.log.warn(e)
		
			# and wait for next iteration
			while not self.should_stop() and time.time()-now < config['Poll']:
				time.sleep(1)

    def submit(self, **opts):  # called on client, so job_info is Job object
        """Log submit event on client."""
        # if this job has a master and it is the first subjob then sent
        # submitted for master job
        if self.job_info.master is not None:
            if self.job_info.id == 0:
                masterjob_msg = self.getMessage('submitted')
                masterjob_msg['subjobs'] = len(self.job_info.master.subjobs)
                masterjob_msg['ganga_job_id'] = str(
                    masterjob_msg['ganga_job_id']).split('.')[0]
                # override ganga_job_uuid as the message 'from the master' is
                # really sent from the subjob
                masterjob_msg['ganga_job_uuid'] = masterjob_msg[
                    'ganga_master_uuid']
                masterjob_msg['ganga_master_uuid'] = 0
                self.send(masterjob_msg)

        from Ganga.Utility import Config
        gangausername = Config.getConfig('Configuration')['user']
        self.job_info.info.monitoring_links.append(
            ('http://gangamon.cern.ch/ganga/#user=%s' % gangausername, 'dashboard'))

        # send submitted for this job
        msg = self.getMessage('submitted')
        from Ganga.GPI import queues
        queues.add(self.send, (msg))

def _initconfig():
    """Initialize MSGMS configuration."""
    try:
        from Ganga.Utility import Config
        # create configuration
        config = Config.makeConfig(
            'MSGMS', 'Settings for the MSGMS monitoring plugin. Cannot be changed ruding the interactive Ganga session.')
        config.addOption(
            'server', 'dashb-mb.cern.ch', 'The server to connect to')
        config.addOption('port', 61113, 'The port to connect to')
        config.addOption('username', 'ganga', '')
        config.addOption('password', 'analysis', '')
        config.addOption('message_destination', '/queue/ganga.status', '')
        config.addOption('usage_message_destination', "/queue/ganga.usage", '')
        config.addOption(
            'job_submission_message_destination', "/queue/ganga.jobsubmission", '')

        # prevent modification during the interactive ganga session
        def deny_modification(name, x):
            raise Config.ConfigError(
                'Cannot modify [MSGMS] settings (attempted %s=%s)' % (name, x))
        config.attachUserHandler(deny_modification, None)
    except ImportError:
        # on worker node so Config is not needed since it is copied to MSGMS
        # constructor
        pass

def _initconfig():
    """Initialize DashboardMS configuration."""
    try:
        from Ganga.Utility import Config
        # create configuration
        config = Config.makeConfig(
            'DashboardMS', 'Settings for Dashboard Messaging Service.')
        config.addOption('server', 'dashb-mb.cern.ch', 'The MSG server name.')
        config.addOption('port', 61113, 'The MSG server port.')
        config.addOption('user', 'ganga-atlas', '')
        config.addOption('password', 'analysis', '')
        config.addOption('destination_job_status', '/topic/dashboard.atlas.jobStatus',
                         'The MSG destination (topic or queue) for job status messages.')
        config.addOption('destination_job_processing_attributes', '/topic/dashboard.atlas.jobProcessingAttributes',
                         'The MSG destination (topic or queue) for job processing attributes messages.')
        config.addOption('destination_job_meta', '/topic/dashboard.atlas.jobMeta',
                         'The MSG destination (topic or queue) for job meta messages.')
        config.addOption('destination_task_meta', '/topic/dashboard.atlas.taskMeta',
                         'The MSG destination (topic or queue) for task meta messages.')
        config.addOption(
            'task_type', 'analysis', 'The type of task. e.g. analysis, production, hammercloud,...')
        # prevent modification during the interactive ganga session

        def deny_modification(name, value):
            raise Config.ConfigError(
                'Cannot modify [DashboardMS] settings (attempted %s=%s)' % (name, value))
        config.attachUserHandler(deny_modification, None)
    except ImportError:
        # on worker node so Config is not needed since it is copied to
        # DashboardMS constructor
        pass

 def getJobInfo(self):  # called on client, so job_info is Job object
     """Create job_info from Job object."""
     if self.job_info.master is None:  # no master job; this job is not splitjob
         ganga_job_id = str(self.job_info.id)
         ganga_job_uuid = self.job_info.info.uuid
         ganga_master_uuid = 0
     else:  # there is a master job; we are in a subjob
         ganga_job_id = str(self.job_info.master.id) + \
             '.' + str(self.job_info.id)
         ganga_job_uuid = self.job_info.info.uuid
         ganga_master_uuid = self.job_info.master.info.uuid
     from Ganga.Utility import Config
     return {'ganga_job_uuid': ganga_job_uuid, 'ganga_master_uuid': ganga_master_uuid, 'ganga_user_repository': Config.getConfig('Configuration')['user']
             + '@' + Config.getConfig('System')['GANGA_HOSTNAME']
             # place-holder updated in getMessage
             # place-holder updated in getMessage
             + ':' + Config.getConfig('Configuration')['gangadir'], 'ganga_job_id': ganga_job_id, 'subjobs': len(self.job_info.subjobs), 'backend': self.job_info.backend.__class__.__name__, 'application': self.job_info.application.__class__.__name__, 'job_name': self.job_info.name, 'hostname': '', 'event': ''
             }

	def __init__(self):
		GangaThread.__init__(self, 'LGI_Resource')
		self.log = getLogger('LGI.Resource.Thread')
		config = Config.getConfig('LGI')
		if not os.path.exists(config['PilotDist']):
			self.log.error('cannot connect to LGI server: pilotjob tarball not found: '+config['PilotDist'])
		self.res = LGI.Resource(config['PilotDist'])
		# number of queued LGI jobs
		self.queued = None

 def start(self):
     config = Config.getConfig("LGI")
     if config["StatsInterval"] == 0:
         self.log.debug("Not starting LGI stats thread because [LGI]StatsInterval is zero")
         return
     if not config["StatsFile"]:
         self.log.debug("Not starting LGI stats thread because [LGI]StatsFile is empty")
         return
     if config["Enable"] is False:
         self.log.debug("Not starting LGI stats thread because [LGI]Enable is False")
         return False
     return GangaThread.start(self)

    def prepare_job_config(self, job):
        """ Generates a CRAB config object from the Ganga job configuration. """

        from WMCore.Configuration import Configuration
        job_config = Configuration()

        for section in job.backend.CRABConfig._schema.datadict.keys():

            section_config = getattr(job.backend.CRABConfig, section)
            ganga_section_config = Config.getConfig('CRABConfig_%s' % section)
            task_section_config = job_config.section_(section)

            for parameter_name, parameter_type in section_config._schema.allItems():
                parameter_value = getattr(section_config, parameter_name)

                if parameter_value not in (None, [None]):
                    
                    # CRAB Config doesn't like Ganga sequence type instead of Lists
                    if parameter_type._meta['sequence']:
                        parameter_value = list(parameter_value)

                    task_section_config.__setattr__(parameter_name, parameter_value)

                # Updating configuration in case of Ganga inline options specified                
                ganga_option = ganga_section_config[parameter_name]
                if ganga_option:

                    # CRAB Config doesn't like Ganga sequence (or tuples) type instead of Lists 
                    # Passing sequance with Ganga inline options makes it a tuple.                   
                    if parameter_type._meta['sequence']:
                        # Ugly but we need this because otherwise tuple of 1 element with a string would
                        # be transformed in a list of chars ( ('ab') --> ['a', 'b'] )
                        import json
                        ganga_option = json.loads(json.dumps(ganga_option))
                        if type(ganga_option) != list:
                            ganga_option = [ganga_option]
                        # loads makes strings 'utf' type, CRAB3 Server wants 'str' type
                        ganga_option = map(lambda x: str(x), ganga_option)

                    task_section_config.__setattr__(parameter_name, ganga_option)

        # Some internal configuration
        job_config.General.workArea = job.outputdir 

        return job_config

def _initconfigFeed():
    """Initialize Feedback configuration."""
    try:
        from Ganga.Utility import Config
        # create configuration
        config = Config.makeConfig(
            'Feedback', 'Settings for the Feedback plugin. Cannot be changed ruding the interactive Ganga session.')
        config.addOption(
            'uploadServer', 'http://gangamon.cern.ch/django/errorreports', 'The server to connect to')

        def deny_modification(name, x):
            raise Config.ConfigError(
                'Cannot modify [Feedback] settings (attempted %s=%s)' % (name, x))
        config.attachUserHandler(deny_modification, None)
    except ImportError:
        # on worker node so Config is not needed since it is copied to Feedback
        # constructor
        pass

    def run(self):
        config = Config.getConfig("LGI")

        # wait for GPI to become ready (taken from GangaJEM)
        while not self.should_stop():
            try:
                from Ganga.GPI import jobs

                break
            except:
                pass
            time.sleep(1)
        from Ganga.GPI import LGI

        # LGI update loop
        self.log.debug("Starting LGI StatsThread main loop")
        self.data = []
        self._writeStats(self.data, config["StatsFile"])
        while not self.should_stop():
            now = time.time()

            try:
                # add new line of data
                lgiQueued = LGI.resource.queued
                if lgiQueued is None:
                    lgiQueued = 0
                lgiRunning = 0  # TODO
                pilotQueued = sum([len(jobs.select(status=s)) for s in ["submitted", "submitting"]])
                pilotRunning = len(jobs.select(status="running"))
                self.data.append(
                    [int(now), lgiQueued + lgiRunning, lgiRunning, pilotQueued + pilotRunning, pilotRunning]
                )
                # trash too old lines
                self.data = filter(lambda x: now - x[0] < config["StatsHistory"], self.data)
                # write data
                self._writeStats(self.data, config["StatsFile"])
            except Exception, e:
                self.log.warn(e)

                # and wait for next iteration
            while not self.should_stop() and time.time() - now < config["StatsInterval"]:
                time.sleep(1)

    def checkReport(self,jobDoc):

        job = self.getJobObject()     

        config = Config.getConfig('Metrics')
        location = config['location']
        if not os.path.exists(location):
            raise BackendError(0,'Location %s file doesnt exist.'%(location))

        config = ConfigParser()
        config.read(location)      

        PARAMS = [('status','status')]

        if config.has_section('report'):
            PARAMS += config.items('report')
        else:
            logger.warning('No report in metrics')

        for n,v in PARAMS:
            if v:
                job.backend.report[v] = jobDoc.getAttribute(v)

    def parseResults(self):

        job = self.getJobObject()   

        server = CRABServer()
        try:
            server.status(job)
            server.getOutput(job) 
        except:
            logger.error('Could not get the output of the job.')
            # Let's not raise this yet (in case of a double call).
            # raise CRABServerError('Impossible to get the output of the job')

        workdir = job.inputdata.ui_working_dir
        index = int(job.id) + 1
        doc_path = '%s/res/crab_fjr_%d.xml'%(workdir,index)

        if not os.path.exists(doc_path):
            logger.error('FJR %s not found.'%(doc_path))
            return

        try:
            doc = parse(doc_path)   
        except:
            logger.error("Could not parse document. File not present?")
            return
        status = doc.firstChild.getAttribute("Status")

        if status in ["Failed"]:
            self.postMortem(job)
            job.updateStatus('failed')
        elif status in ["Success"]:
            if job.status == 'submitting':
                job.updateStatus('submitted')
            job.updateStatus('completed')
        else:
            logger.warning("UNKNOWN PARSE STATUS: "+str(status))

        config = Config.getConfig('Metrics')
        location = config['location']
        if not os.path.exists(location):
            raise BackendError(0,'Location %s file doesnt exist.'%(location))

        config = ConfigParser()
        config.read(location)      

        #Iterate over all them
        SECTIONS = config.sections()
        if 'report' in SECTIONS:
            SECTIONS.remove('report')

        # Only five sections work here...
        for section in SECTIONS:

            if not job.backend.fjr.has_key(section):
                job.backend.fjr[section] = {}

            performancereport = doc.getElementsByTagName("PerformanceReport")[0]
            performancesummary = performancereport.getElementsByTagName("PerformanceSummary")
            for pfs in performancesummary:
                if pfs.getAttribute("Metric") == section:
                    metrics = pfs.getElementsByTagName("Metric")
                    for metric in metrics:
                        name = metric.getAttribute("Name")
                        if config.has_option(section,name):
                            # Due to the names with minus intead of underscore, we have to do thiw walkarround
                            # to send them to the DB.
                            name = config.get(section,name)
                            if name:
                                job.backend.fjr[section][name] = metric.getAttribute("Value")

    def report_inner(job=None, isJob=False, isTask=False):

        userInfoDirName = "userreport"
        tempDirName = "reportsRepository"
        # job relevant info
        jobSummaryFileName = "jobsummary.txt"
        jobFullPrintFileName = "jobfullprint.txt"
        repositoryPath = "repository/$usr/LocalXML/6.0/jobs/$thousandsNumxxx"
        # task relevant info
        taskSummaryFileName = "tasksummary.txt"
        taskFullPrintFileName = "taskfullprint.txt"
        tasksRepositoryPath = "repository/$usr/LocalXML/6.0/tasks/$thousandsNumxxx"
        # user's info
        environFileName = "environ.txt"
        userConfigFileName = "userconfig.txt"
        defaultConfigFileName = "gangarc.txt"
        ipythonHistoryFileName = "ipythonhistory.txt"
        gangaLogFileName = "gangalog.txt"
        jobsListFileName = "jobslist.txt"
        tasksListFileName = "taskslist.txt"
        from Ganga.Utility import Config
        uploadFileServer = Config.getConfig('Feedback')['uploadServer']
        #uploadFileServer= "http://gangamon.cern.ch/django/errorreports/"
        #uploadFileServer= "http://ganga-ai-02.cern.ch/django/errorreports/"
        #uploadFileServer= "http://127.0.0.1:8000/errorreports"

        def printDictionary(dictionary, file=sys.stdout):
            for k, v in dictionary.iteritems():
                print('%s: %s' % (k, v), file=file)

                if k == 'PYTHONPATH':
                    global PYTHON_PATH
                    PYTHON_PATH = v

        def extractFileObjects(fileName, targetDirectoryName):
            try:
                fileToRead = open(fileName, 'r')
                try:
                    fileText = fileToRead.read()
                    import re
                    pattern = "File\(name=\'(.+?)\'"
                    matches = re.findall(pattern, fileText)

                    for fileName in matches:
                        fileName = os.path.expanduser(fileName)
                        targetFileName = os.path.join(
                            targetDirectoryName, os.path.basename(fileName))
                        shutil.copyfile(fileName, targetFileName)

                finally:
                    fileToRead.close()
            # except IOError, OSError:
            except Exception as err:
                logger.debug("Err: %s" % str(err))
                writeErrorLog(str(sys.exc_info()[1]))

        def writeErrorLog(errorMessage):
            try:
                fileToWrite = open(errorLogPath, 'a')
                try:
                    fileToWrite.write(errorMessage)
                    fileToWrite.write("\n")
                except Exception as err:
                    logger.debug("Err: %s" % str(err))
                    raise
                finally:
                    fileToWrite.close()
            except Exception as err2:
                logger.debug("Err: %s" % str(err2))
                pass

        def writeStringToFile(fileName, stringToWrite):

            try:
                # uncomment this to try the error logger
                #fileName = '~/' + fileName
                fileToWrite = open(fileName, 'w')
                try:
                    fileToWrite.write(stringToWrite)
                except Exception as err:
                    logger.debug("Err: %s" % str(err))
                    raise err
                finally:
                    fileToWrite.close()
            # except IOError:
            except Exception as err:
                logger.debug("Err2: %s" % str(err))
                writeErrorLog(str(sys.exc_info()[1]))

        def renameDataFiles(directory):

            for fileName in os.listdir(directory):
                fullFileName = os.path.join(directory, fileName)
                if os.path.isfile(fullFileName):
                    if fileName == 'data':
                        os.rename(fullFileName, fullFileName + '.txt')
                else:
                    renameDataFiles(fullFileName)

        import shutil
#.........这里部分代码省略.........

#
# This is free software; you can redistribute it and/or modify it under the terms of
# the GNU General Public License as published by the Free Software Foundation.
#
# http://www.gnu.org/licenses/gpl.txt

import os
import time
import datetime
from InterpoList import InterpoList
from Ganga.Utility import Config
from Ganga.Utility.logging import getLogger
from Ganga.Core.GangaThread import GangaThread

# Module configuration
config = Config.makeConfig('LGI', 'Leiden Grid Initiative Pilot job framework settings')
config.addOption('PilotDist', 'pilotdist/pilotjob.tar.gz',
	'Pilot job resource daemon tarball, fully configured for your applications and project server')
config.addOption('PilotScript', 'pilotdist/pilotrun.sh',
	'Script to run inside pilotjob, which unpacks the tarball and executes the resource daemon')

config.addOption('SchedMin',  1, 'Minimum number of pilotjobs at all times')
config.addOption('SchedMax', 10, 'Maximum number of pilotjobs')

config.addOption('Poll', 30, 'LGI thread polling time')
config.addOption('Update', 10, 'Pilot thread update time')
config.addOption('WaitNew', 60, 'If after this many seconds there are (still) more LGI jobs than pilotjobs, spawn new pilotjobs.')
config.addOption('WaitTerm', 300, 'Terminate pilotjob after seconds of idle time')
config.addOption('MaxRuntime', None, 'Maximum run-time of pilotjobs in seconds. Leave empty to run indefinitely until batch system terminates it.')

config.addOption('StatsInterval', 0, 'Statistics logging interval, or 0 for no statistics')