Python pandatools.Client类代码示例

def addDatasetsToContainer(container, datasets):
    from pandatools import Client

    # HC datasets don't use containers
    if not configPanda["processingType"].startswith("gangarobot") and not configPanda["processingType"].startswith(
        "hammercloud"
    ):
        Client.addDatasetsToContainer(container, datasets, False)

    def master_resubmit(self,jobs):
        '''Resubmit failed Jedi job'''
        from pandatools import Client

        jobIDs = {}
        for job in jobs: 
            jobIDs[job.backend.id] = job

        allJobIDs = jobIDs.keys()
        pandaJobIDs = {}
        for jID in allJobIDs:
            with inject_proxy(self.credential_requirements):
                status, jediTaskDict = Client.getJediTaskDetails({'jediTaskID': jID},False,True,verbose=False)
            if status != 0:
                logger.error("Failed to get task details for %s" % jID)
                raise BackendError('Jedi','Return code %d retrieving job status information.' % status)

            # Retrieve job
            job = jobIDs[jediTaskDict['jediTaskID']]
       
            newJobsetID = -1 # get jobset
            retryJobs = [] # jspecs
            resubmittedJobs = [] # ganga jobs

            if jediTaskDict['status'] in ['failed', 'killed', 'cancelled', 'aborted', 'broken', 'finished' ]:
                retryJobs.append(job)
                resubmittedJobs.append(jID)
            #elif jediTaskDict['status'] == 'finished':
            #    pass
            else:
                logger.warning("Cannot resubmit. Jedi task %s is status %s." %(jID, jediTaskDict['status'] ))
                return False

            # submit
            if len(retryJobs)==0:
                logger.warning("No failed jobs to resubmit")
                return False

            with inject_proxy(self.credential_requirements):
                status,out = Client.retryTask(jID, verbose=False)
            if status != 0:
                logger.error(status)
                logger.error(out)
                logger.error("Failed to retry JobID=%s" % jID)                                                                                         
                return False
            tmpStat,tmpDiag = out
            if not tmpStat:
                logger.error(tmpDiag)
                logger.error("Failed to retry JobID=%s" % jID)
                return False
            logger.info(tmpDiag)
       
            job.backend.status = None
            job.backend.jobSpec = {}
            job.updateStatus('submitted')

        logger.info('Resubmission successful')
        return True

def createContainer(name):
    from pandatools import Client
    # don't create containers for HC datasets
    if not configPanda['processingType'].startswith('gangarobot') and not configPanda['processingType'].startswith('hammercloud'):
        try:
            Client.createContainer(name,False)
            logger.info('Created output container %s' %name)
        except exceptions.SystemExit:
            raise BackendError('Panda','Exception in Client.createContainer %s: %s %s'%(name,sys.exc_info()[0],sys.exc_info()[1]))

def createContainer(name):
    from pandatools import Client

    # don't create containers for HC datasets
    if not configPanda["processingType"].startswith("gangarobot") and not configPanda["processingType"].startswith(
        "hammercloud"
    ):
        try:
            Client.createContainer(name, False)
            logger.info("Created output container %s" % name)
        except exceptions.SystemExit:
            raise BackendError(
                "Panda", "Exception in Client.createContainer %s: %s %s" % (name, sys.exc_info()[0], sys.exc_info()[1])
            )

def getDBDatasets(jobO,trf,dbrelease):
    from pandatools import Client

    # get DB datasets
    dbrFiles  = {}
    dbrDsList = []
    if trf or dbrelease != '':
        if trf:
            # parse jobO for TRF
            tmpItems = jobO.split()
        else:
            # mimic a trf parameter to reuse following algorithm
            tmpItems = ['%DB='+dbrelease]
        # look for DBRelease
        for tmpItem in tmpItems:
            match = re.search('%DB=([^:]+):(.+)$',tmpItem)
            if match:
                tmpDbrDS  = match.group(1)
                tmpDbrLFN = match.group(2)
                # get files in the dataset
                if not tmpDbrDS in dbrDsList:
                    logger.info("Querying files in dataset:%s" % tmpDbrDS)
                    try:
                        tmpList = Client.queryFilesInDataset(tmpDbrDS,False)
                    except:
                        raise ApplicationConfigurationError("ERROR : error while looking up dataset %s. Perhaps this dataset does not exist?"%tmpDbrDS)
                    # append
                    for tmpLFN,tmpVal in tmpList.iteritems():
                        dbrFiles[tmpLFN] = tmpVal
                    dbrDsList.append(tmpDbrDS)
                # check
                if tmpDbrLFN not in dbrFiles:
                    raise ApplicationConfigurationError("ERROR : %s is not in %s"%(tmpDbrLFN,tmpDbrDS))
    return dbrFiles,dbrDsList

def getLatestDBReleaseCaching():
    import tempfile
    import cPickle as pickle
    from pandatools import Client
    from GangaAtlas.Lib.Credentials.ProxyHelper import getNickname

    TMPDIR = tempfile.gettempdir()
    nickname = getNickname(allowMissingNickname=False)
    DBRELCACHE = '%s/ganga.latestdbrel.%s'%(TMPDIR,nickname)

    try:
        fh = open(DBRELCACHE)
        dbrelCache = pickle.load(fh)
        fh.close()
        if dbrelCache['mtime'] > time.time() - 3600:
            logger.debug('Loading LATEST DBRelease from local cache')
            return dbrelCache['atlas_dbrelease']
        else:
            raise Exception()
    except:
        logger.debug('Updating local LATEST DBRelease cache')
        atlas_dbrelease = Client.getLatestDBRelease(False)
        dbrelCache = {}
        dbrelCache['mtime'] = time.time()
        dbrelCache['atlas_dbrelease'] = atlas_dbrelease
        fh = open(DBRELCACHE,'w')
        pickle.dump(dbrelCache,fh)
        fh.close()
        return atlas_dbrelease

  def convertQueueNameToDQ2Names(self, queue, test):
    from pandatools import Client
    from dq2.info.TiersOfATLAS import ToACache
    sites = []

    if 'PFT' in test.template.description:
        #Client.PandaSites = Client.getSiteSpecs('production')[1]
        Client.PandaSites = Client.getSiteSpecs('all')[1]

    for site in Client.PandaSites[queue]['setokens'].values():
      sites.append(Client.convSrmV2ID(site))
    allowed_sites = []
    for site in ToACache.sites:
      if (site not in allowed_sites
          and Client.convSrmV2ID(site) in sites
          and site.find('TAPE') == -1 and 'DISK' in site):
        allowed_sites.append(site)
    return allowed_sites

    def master_submit(self,rjobs,subjobspecs,buildjobspec):
        '''Submit jobs'''
       
        from pandatools import Client
        from pandatools import MiscUtils

        from Ganga.Core import IncompleteJobSubmissionError
        from Ganga.Utility.logging import log_user_exception

        job = self.getJobObject()

        # job name
        jobName = 'ganga.%s' % MiscUtils.wrappedUuidGen()

        jobspecs = {}
        if buildjobspec:
            jobspecs = buildjobspec
        else:
            jobspecs = subjobspecs

        logger.debug(jobspecs)

        # submit task
        for subjob in rjobs:
            subjob.updateStatus('submitting')

        logger.info("Submitting to Jedi ...")
        verbose = logger.isEnabledFor(10)
        with inject_proxy(self.credential_requirements):
            status, tmpOut = Client.insertTaskParams(jobspecs, verbose)

        logger.debug(tmpOut)

        if status != 0:
            logger.error("Task submission to Jedi failed with %s " %status)
            return False
        if tmpOut[0] == False:
            logger.error("Task submission to Jedi failed %s" %tmpOut[1])
            return False
        logger.info("Task submission to Jedi suceeded with new jediTaskID=%s" %tmpOut[1])

        #if buildjobspec:
        #    job.backend.buildjob = PandaBuildJob() 
        #    job.backend.buildjob.id = jobids[0][0]
        #    job.backend.buildjob.url = 'http://panda.cern.ch/?job=%d'%jobids[0][0]
        #    del jobids[0]

        for subjob in rjobs:
            subjob.backend.id = tmpOut[1]
            subjob.backend.url = 'http://pandamon.cern.ch/jedi/taskinfo?days=20&task=%d'%tmpOut[1]
            subjob.updateStatus('submitted')
            logger.info("Panda monitor url: %s" %subjob.backend.url)

        return True

	def getJobIDs(self, timestamp):
		date = time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime(timestamp))

		status, output = Client.getJobIDsInTimeRange(date)

		if status != 0:
			print 'Error: panda !'
			return []

		output.sort()

		return output

 def finish(self,JobID,soft=False):
     # get logger
     tmpLog = PLogger.getPandaLogger()
     # check proxy
     self.gridPassPhrase,self.vomsFQAN = PsubUtils.checkGridProxy(
             self.gridPassPhrase,
             False,
             self.verbose,
             useCache=True)
     # force update just in case
     self.status(JobID,True)
     # get jobset
     jobList = self.getJobIDsWithSetID(JobID)
     if jobList == None:
         # works only for jobsetID
         if useJobsetID:
             return
         # works with jobID
         jobList = [JobID]
     else:
         tmpMsg = "ID=%s is composed of JobID=" % JobID
         for tmpJobID in jobList:
             tmpMsg += '%s,' % tmpJobID
         tmpMsg = tmpMsg[:-1]
         tmpLog.info(tmpMsg)
     for tmpJobID in jobList:    
         # get job info from local repository
         job = self.getJobInfo(tmpJobID)
         if job == None:
             tmpLog.warning("JobID=%s not found in local repository. Synchronization may be needed" % tmpJobID)            
             continue
         # skip frozen job
         if job.dbStatus == 'frozen':
             tmpLog.info('All subJobs in JobID=%s already finished/failed' % tmpJobID)
             continue
         # finish JEDI task
         tmpLog.info('Sending finishTask command ...')
         status,output = Client.finishTask(job.jediTaskID,soft,self.verbose)
         # communication error
         if status != 0:
             tmpLog.error(output)
             tmpLog.error("Failed to finish JobID=%s" % tmpJobID)
             return False
         tmpStat,tmpDiag = output
         if not tmpStat:
             tmpLog.error(tmpDiag)
             tmpLog.error("Failed to finish JobID=%s" % tmpJobID)
             return False
         tmpLog.info(tmpDiag)
     # done
     tmpLog.info('Done. TaskID=%s will be finished soon' % job.jediTaskID)
     return True

def retrievePandaJobs(job, jIDs):
    '''
    methods for retrieving panda job ids of panda jobs given a jobDefId
    '''
    from pandatools import Client

    ick       = False
    jstatus    = ''
    num_pjobs = 0

    logger.debug("retrievePandaJobs jIDs=%s" %jIDs)

    # get status from Panda server
    rc, jobsStatus = Client.getFullJobStatus(jIDs,False)
    if rc:
        logger.error('Return code %d retrieving job status information.',rc)
        raise BackendError('Jedi','Return code %d retrieving job status information.' % rc)

    for status in jobsStatus:
        if not status: continue

        jstatus = status.jobStatus
        if status.jobStatus == None:
            logger.warning('No panda jobs expected')
            job.backend.pandajobs = []

        elif status.jobStatus in [ "defined", "activated", "running", "failed", "finished", "holding", "assigned"]:
            logger.debug('Panda jobs are running')
            logger.debug("PandaID: %d" % status.PandaID)

            pjobj         = JediPandaJob()
            pjobj.id      = status.PandaID
            pjobj.url     = 'http://panda.cern.ch/?job=%d' % status.PandaID
            pjobj.jopSpec = dict(zip(status._attributes,status.values()))
            for k in pjobj.jobSpec.keys():
                if type(pjobj.jobSpec[k]) not in [type(''),type(1)]:
                    pjobj.jobSpec[k]=str(pjobj.jobSpec[k])

            if pjobj not in job.backend.pandajobs:
                job.backend.pandajobs.append(pjobj)
            else:
                logger.debug("Panda job %s already exists locally" % pjobj.id)
                
            num_pjobs += 1
        else:
            logger.warning("getFullJobStatus returned unsupported status %s for Panda job %s " %(status.jobStatus, status.PandaID) )
            
        ick = True

    return (ick, jstatus, num_pjobs)

	def jobIsCancelled(self, ID):
		L = Client.getPandIDsWithJobID(ID)[1]

		if L is None:
			result = False
		else:
			result = False

			for item in L:
				if L[item][0] == 'cancelled':
					result = True
					pass

		return result

	def jobIsFinished(self, ID):
		L = Client.getPandIDsWithJobID(ID)[1]

		if L is None:
			result = False
		else:
			result = True

			for item in L:
				if L[item][0] != 'finished':
					result = False
					pass

		return result

 def debug(self,pandaID,modeOn):
     # get logger
     tmpLog = PLogger.getPandaLogger()
     # check proxy
     self.gridPassPhrase,self.vomsFQAN = PsubUtils.checkGridProxy(
         self.gridPassPhrase,
         False,
         self.verbose,
         useCache=True)
     # rebrokerage
     status,output = Client.setDebugMode(pandaID,modeOn,self.verbose)
     if status != 0:
         tmpLog.error(output)
         tmpLog.error("Failed to set debug mode for %s" % pandaID)
         return
     # done
     tmpLog.info(output)
     return

  def change_site_status(self, site, new_status):
    if new_status not in ('test', 'online'):
      return False
    self.add_log('Changing %s status to %s:' % (site, new_status))
    if self.debug:
      self.add_log('DEBUG mode')
      return True
    now = int(time.time())
    if new_status == 'test':
      try:
        last_exclusion = int(Site.objects.filter(name=site)[0].getSiteOptions_for_site.filter(option_name='last_exclusion')[0].option_value)
      except:
        last_exclusion = 0
      if now - last_exclusion < 21600:
        self.add_log('%s was recently auto-excluded. Skipping...' % site)
        return False

    cmd = "curl -s -k --cert $X509_USER_PROXY 'https://panda.cern.ch:25943/server/controller/query?tpmes=setmanual&queue=%s&moduser=HammerCloud&comment=HC.Blacklist.set.manual'" % site
    self.add_log('> ' + cmd)
    if not self.debug:
      o = commands.getoutput(cmd)
      self.add_log(o)
    cmd = "curl -s -k --cert $X509_USER_PROXY 'https://panda.cern.ch:25943/server/controller/query?tpmes=set%s&queue=%s&moduser=HammerCloud&comment=HC.Blacklist.set.%s'" % (new_status, site, new_status)
    self.add_log('> ' + cmd)
    if not self.debug:
      o = commands.getoutput(cmd)
      self.add_log(o)
    if new_status == 'test':
      try:
        option = Site.objects.filter(name=site)[0].getSiteOptions_for_site.filter(option_name='last_exclusion')[0]
        option.option_value = now
      except:
        option = SiteOption()
        option.option_name = 'last_exclusion'
        option.option_value = now
        option.site = Site.objects.get(name=site)
      option.save()

    Client.PandaSites = Client.getSiteSpecs(SITETYPE)[1]
    if not self.debug and Client.PandaSites[site]['status'] != new_status:
      self.store_log('Error setting %s to %s' % (site, new_status), 'error')
      return False
     
    return True