Python Client.get_replication_rule方法代码示例

# 需要导入模块: from rucio.client import Client [as 别名]
# 或者: from rucio.client.Client import get_replication_rule [as 别名]
 def check_status(self, jobspec):
     # make logger
     tmpLog = self.make_logger(baseLogger, 'PandaID={0}'.format(jobspec.PandaID),
                               method_name='check_status')
     tmpLog.debug('start')
     # loop over all files
     allChecked = True
     oneErrMsg = None
     transferStatus = dict()
     for fileSpec in jobspec.outFiles:
         # skip already don
         if fileSpec.status in ['finished', 'failed']:
             continue
         # get transfer ID
         transferID = fileSpec.fileAttributes['transferID']
         if transferID not in transferStatus:
             # get status
             try:
                 rucioAPI = RucioClient()
                 ruleInfo = rucioAPI.get_replication_rule(transferID)
                 tmpTransferStatus = ruleInfo['state']
                 tmpLog.debug('got state={0} for rule={1}'.format(tmpTransferStatus, transferID))
             except RuleNotFound:
                 tmpLog.error('rule {0} not found'.format(transferID))
                 tmpTransferStatus = 'FAILED'
             except:
                 err_type, err_value = sys.exc_info()[:2]
                 errMsg = "{0} {1}".format(err_type.__name__, err_value)
                 tmpLog.error('failed to get status for rule={0} with {1}'.format(transferID, errMsg))
                 # set dummy not to lookup again
                 tmpTransferStatus = None
                 allChecked = False
                 # keep one message
                 if oneErrMsg is None:
                     oneErrMsg = errMsg
             tmpTransferStatus = 'OK'
             transferStatus[transferID] = tmpTransferStatus
         # final status
         if transferStatus[transferID] == 'OK':
             fileSpec.status = 'finished'
         elif transferStatus[transferID] in ['FAILED', 'CANCELED']:
             fileSpec.status = 'failed'
     if allChecked:
         return True, ''
     else:
         return False, oneErrMsg

# 需要导入模块: from rucio.client import Client [as 别名]
# 或者: from rucio.client.Client import get_replication_rule [as 别名]
    def check_status(self, jobspec):
        tmpStat = True
        tmpMsg = ''
        # make logger
        tmpLog = self.make_logger(baseLogger, 'PandaID={0} ThreadID={1}'.format(jobspec.PandaID,threading.current_thread().ident),
                                  method_name='check_status')
        tmpLog.debug('start')
        # check that jobspec.computingSite is defined
        if jobspec.computingSite is None:
            # not found
            tmpLog.error('jobspec.computingSite is not defined')
            return False, 'jobspec.computingSite is not defined'
        else:
            tmpLog.debug('jobspec.computingSite : {0}'.format(jobspec.computingSite))
        # get the queueConfig and corresponding objStoreID_ES
        queueConfigMapper = QueueConfigMapper()
        queueConfig = queueConfigMapper.get_queue(jobspec.computingSite)
        # write to debug log queueConfig.stager
        tmpLog.debug('jobspec.computingSite - {0} queueConfig.stager {1}'.format(jobspec.computingSite,queueConfig.stager))
        # check queueConfig stager section to see if jobtype is set
        if 'jobtype' in queueConfig.stager:
            if queueConfig.stager['jobtype'] == "Yoda" :
                self.Yodajob = True
        # get destination endpoint
        nucleus = jobspec.jobParams['nucleus']
        agis = self.dbInterface.get_cache('panda_queues.json').data
        dstRSE = [agis[x]["astorages"]['pr'][0] for x in agis if agis[x]["atlas_site"] == nucleus][0]
        # set the location of the files in fileSpec.objstoreID
        # see file /cvmfs/atlas.cern.ch/repo/sw/local/etc/agis_ddmendpoints.json 
        ddm = self.dbInterface.get_cache('agis_ddmendpoints.json').data
        self.objstoreID = ddm[dstRSE]['id']
        if self.Yodajob :
            self.pathConvention = int(queueConfig.stager['pathConvention'])
            tmpLog.debug('Yoda Job - PandaID = {0} objstoreID = {1} pathConvention ={2}'.format(jobspec.PandaID,self.objstoreID,self.pathConvention))
        else:
            self.pathConvention = None
            tmpLog.debug('PandaID = {0} objstoreID = {1}'.format(jobspec.PandaID,self.objstoreID))
        # set the location of the files in fileSpec.objstoreID
        self.set_FileSpec_objstoreID(jobspec, self.objstoreID, self.pathConvention)
        # Get the files grouped by Rucio Rule ID 
        groups = jobspec.get_groups_of_output_files()
        if len(groups) == 0:
            tmpLog.debug('No Rucio Rules')
            return None,'No Rucio Rules'
        tmpLog.debug('#Rucio Rules - {0} - Rules - {1}'.format(len(groups),groups)) 
        
        try:
            rucioAPI = RucioClient()
        except:
            tmpLog.error('failure to get Rucio Client try again later')
            return None,'failure to get Rucio Client try again later'

        # loop over the Rucio rules 
        for rucioRule in groups:
            if rucioRule is None:
                continue
            # lock
            have_db_lock = self.dbInterface.get_object_lock(rucioRule, lock_interval=120)
            if not have_db_lock:
                msgStr = 'escape since {0} is locked by another thread'.format(rucioRule)
                tmpLog.debug(msgStr)
                return None, msgStr
            # get transfer status
            groupStatus = self.dbInterface.get_file_group_status(rucioRule)
            tmpLog.debug('rucioRule - {0} - groupStatus - {1}'.format(rucioRule,groupStatus))
            if 'transferred' in groupStatus:
                # already succeeded - set the fileSpec status for these files 
                self.set_FileSpec_objstoreID(jobspec, self.objstoreID, self.pathConvention)
                pass
            elif 'failed' in groupStatus :
                # transfer failure
                tmpStat = False
                tmpMsg = 'rucio rule for {0}:{1} already failed'.format(datasetScope, datasetName)
            elif 'transferring' in groupStatus or 'pending' in groupStatus:
                # transfer started in Rucio check status
                try:
                    result = rucioAPI.get_replication_rule(rucioRule,False)
                    if result['state'] == "OK" :
                        # files transfered to nucleus
                        tmpLog.debug('Files for Rucio Rule {0} successfully transferred'.format(rucioRule))
                        self.dbInterface.update_file_group_status(rucioRule, 'transferred')
                        # set the fileSpec status for these files 
                        self.set_FileSpec_objstoreID(jobspec, self.objstoreID, self.pathConvention)
                        self.set_FileSpec_status(jobspec,'finished')
                    elif result['state'] == "FAILED" :
                        # failed Rucio Transfer
                        tmpStat = False
                        tmpMsg = 'Failed Rucio Transfer - Rucio Rule - {0}'.format(rucioRule)
                        tmpLog.debug(tmpMsg)
                        self.set_FileSpec_status(jobspec,'failed')
                    elif result['state'] == 'STUCK' :
                        tmpStat = None
                        tmpMsg = 'Rucio Transfer Rule {0} Stuck'.format(rucioRule)
                        tmpLog.debug(tmpMsg)
                except:
                        tmpStat = None
                        tmpMsg = 'Could not get information or Rucio Rule {0}'.format(rucioRule)
                        tmpLog.error(tmpMsg)
                        pass
            # release the lock
#.........这里部分代码省略.........

# 需要导入模块: from rucio.client import Client [as 别名]
# 或者: from rucio.client.Client import get_replication_rule [as 别名]
    def check_status(self, jobspec):
        tmpStat = True
        tmpMsg = ''
        # make logger
        tmpLog = self.make_logger(baseLogger, 'PandaID={0} ThreadID={1}'.format(jobspec.PandaID,threading.current_thread().ident),
                                  method_name='check_status')
        tmpLog.debug('start')
        # Get the files grouped by Rucio Rule ID 
        groups = jobspec.get_groups_of_output_files()
        if len(groups) == 0:
            tmpLog.debug('No Rucio Rules')
            return None,'No Rucio Rules'
        tmpLog.debug('#Rucio Rules - {0} - Rules - {1}'.format(len(groups),groups))
        
        try:
            rucioAPI = RucioClient()
        except:
            tmpLog.error('failure to get Rucio Client try again later')
            return None,'failure to get Rucio Client try again later'

        # loop over the Rucio rules 
        for rucioRule in groups:
            if rucioRule is None:
                continue
            # lock
            have_db_lock = self.dbInterface.get_object_lock(rucioRule, lock_interval=120)
            if not have_db_lock:
                msgStr = 'escape since {0} is locked by another thread'.format(rucioRule)
                tmpLog.debug(msgStr)
                return None, msgStr
            # get transfer status
            groupStatus = self.dbInterface.get_file_group_status(rucioRule)
            if 'transferred' in groupStatus :
                # already succeeded
                pass
            elif 'failed' in groupStatus :
                # transfer failure
                tmpStat = False
                tmpMsg = 'rucio rule for {0}:{1} already failed'.format(datasetScope, datasetName)
            elif 'transferring' in groupStatus:
                # transfer started in Rucio check status
                try:
                    result = rucioAPI.get_replication_rule(rucioRule,False)
                    if result['state'] == "OK" :
                        # files transfered to nucleus
                        tmpLog.debug('Files for Rucio Rule {0} successfully transferred'.format(rucioRule))
                        self.dbInterface.update_file_group_status(rucioRule, 'transferred')
                        # set the fileSpec status for these files 
                        self.set_FileSpec_status(jobspec,'finished')
                    elif result['state'] == "FAILED" :
                        # failed Rucio Transfer
                        tmpStat = False
                        tmpMsg = 'Failed Rucio Transfer - Rucio Rule - {0}'.format(rucioRule)
                        tmpLog.debug(tmpMsg)
                        self.set_FileSpec_status(jobspec,'failed')
                    elif result['state'] == 'STUCK' :
                        tmpStat = None
                        tmpMsg = 'Rucio Transfer Rule {0} Stuck'.format(rucioRule)
                        tmpLog.debug(tmpMsg)
                except:
                        tmpStat = None
                        tmpMsg = 'Could not get information or Rucio Rule {0}'.format(rucioRule)
                        tmpLog.error(tmpMsg)
                        pass
            # release the lock
            if have_db_lock:
                tmpLog.debug('attempt to release DB lock for Rucio Rule {0}'.format(rucioRule))
                release_db_lock = self.dbInterface.release_object_lock(rucioRule) 
                if release_db_lock:
                    tmpLog.debug('released DB lock for rucioRule - {0}'.format(rucioRule))
                    have_db_lock = False 
                else:
                    msgStr = ' Could not release DB lock for {}'.format(rucioRule)
                    tmpLog.error(msgStr)
                    return None, msgStr

        tmpLog.debug('stop')
        return  tmpStat, tmpMsg