Python Client.list_files方法代码示例

# 需要导入模块: from rucio.client import Client [as 别名]
# 或者: from rucio.client.Client import list_files [as 别名]
 def listFilesInDataset(self,datasetName,long=False,fileList=None):
     # extract scope from dataset
     scope,dsn = self.extract_scope(datasetName)
     if dsn.endswith('/'):
         dsn = dsn[:-1]
     client = RucioClient()
     return_dict = {}
     for x in client.list_files(scope, dsn, long=long):
         tmpLFN = str(x['name'])
         if fileList != None:
             genLFN = re.sub('\.\d+$','',tmpLFN)
             if not tmpLFN in fileList and not genLFN in fileList:
                 continue
         dq2attrs = {}
         dq2attrs['chksum'] = "ad:" + str(x['adler32'])
         dq2attrs['md5sum'] = dq2attrs['chksum']
         dq2attrs['checksum'] = dq2attrs['chksum']
         dq2attrs['fsize'] = x['bytes']
         dq2attrs['filesize'] = dq2attrs['fsize']
         dq2attrs['scope'] = str(x['scope'])
         dq2attrs['events'] = str(x['events'])
         if long:
             dq2attrs['lumiblocknr'] = str(x['lumiblocknr'])
         guid = str('%s-%s-%s-%s-%s' % (x['guid'][0:8], x['guid'][8:12], x['guid'][12:16], x['guid'][16:20], x['guid'][20:32]))
         dq2attrs['guid'] = guid
         return_dict[tmpLFN] = dq2attrs
     return (return_dict, None)

# 需要导入模块: from rucio.client import Client [as 别名]
# 或者: from rucio.client.Client import list_files [as 别名]
 def getDatasetSize(self,datasetName):
     # extract scope from dataset
     scope,dsn = self.extract_scope(datasetName)
     client = RucioClient()
     tSize = 0
     try:
         for x in client.list_files(scope, dsn, long=long):
             tSize += x['bytes']
         return True,tSize
     except DataIdentifierNotFound:
         return None,'dataset not found'
     except:
         errtype, errvalue = sys.exc_info()[:2]
         errMsg = '{0} {1}'.format(errtype.__name__, errvalue)
         return False,errMsg

# 需要导入模块: from rucio.client import Client [as 别名]
# 或者: from rucio.client.Client import list_files [as 别名]
 def getNumberOfFiles(self,datasetName,presetScope=None):
     # extract scope from dataset
     scope,dsn = self.extract_scope(datasetName)
     if presetScope is not None:
         scope = presetScope
     client = RucioClient()
     nFiles = 0
     try:
         for x in client.list_files(scope, dsn, long=long):
             nFiles += 1
         return True,nFiles
     except DataIdentifierNotFound:
         return None,'dataset not found'
     except:
         errtype, errvalue = sys.exc_info()[:2]
         errMsg = '{0} {1}'.format(errtype.__name__, errvalue)
         return False,errMsg

# 需要导入模块: from rucio.client import Client [as 别名]
# 或者: from rucio.client.Client import list_files [as 别名]
                    help='not retry child tasks')
parser.add_argument('--resurrectDS',action='store_const',const=True,dest='resurrectDS',default=False,
                    help='resurrect output and log datasets if they were already deleted')
parser.add_argument('--dryRun',action='store_const',const=True,dest='dryRun',default=False,
                    help='dry run')

options = parser.parse_args()

if options.files is not None:
    files = options.files.split(',')
else:
    # get files from rucio
    rc = RucioClient()
    scope, name = rucioAPI.extract_scope(options.ds)
    files_rucio = set()
    for i in rc.list_files(scope, name):
        files_rucio.add(i['name'])
    # get files from panda
    dsName = options.ds.split(':')[-1]
    fd,fo = taskBuffer.querySQLS('SELECT c.lfn FROM ATLAS_PANDA.JEDI_Datasets d,ATLAS_PANDA.JEDI_Dataset_Contents c WHERE d.jediTaskID=c.jediTaskID AND d.datasetID=c.datasetID AND d.type IN (:t1,:t2) AND c.status=:s AND d.datasetName=:name ',
                                     {':s': 'finished', ':t1': 'output', ':t2': 'log', ':name': dsName})
    files = []
    for tmpLFN, in fo:
        if tmpLFN not in files_rucio:
            files.append(tmpLFN)
    print
    print 'found {0} lost files -> {1}'.format(len(files), ','.join(files))

s,jediTaskID = taskBuffer.resetFileStatusInJEDI('',True,options.ds,files,[],options.dryRun)
if options.dryRun:
    sys.exit(0)

# 需要导入模块: from rucio.client import Client [as 别名]
# 或者: from rucio.client.Client import list_files [as 别名]

#.........这里部分代码省略.........
    try:
        amiclient = pyAMI.client.Client('atlas')
        amiOutDict = pyAMI.utils.smart_execute(amiclient, 'datasets', [], None, None, None, False, **kwargs).get_rows()
    except:
        errType,errValue = sys.exc_info()[:2]
        tmpLog.error("%s %s" % (errType,errValue))
        tmpLog.error('pyAMI failed')
        return failedRet
    # get dataset map
    if verbose:
        tmpLog.debug(amiOutDict)
    # parse
    rucioclient = RucioClient()
    datasetListFromAMI = []
    runDsMap = dict()
    for tmpVal in amiOutDict:
        if tmpVal.has_key('ldn'):
            dsName = str(tmpVal['ldn'])
            # check dataset names
            if goodRunListDS == []:    
                matchFlag = True
            else:
                matchFlag = False
                for tmpPatt in goodRunListDS:
                    if re.search(tmpPatt,dsName) != None:
                        matchFlag = True
            if not matchFlag:
                continue
            # get metadata
            try:
                tmpLog.debug("getting metadata for %s" % dsName)
                scope,dsn = extract_scope(dsName)
                meta =  rucioclient.get_metadata(scope, dsn)
                if meta['did_type'] == 'COLLECTION':
                    dsName += '/'
                datasetListFromAMI.append(dsName)
                tmpRunNum = meta['run_number']
                tmpLog.debug("run number : %s" % tmpRunNum)
                if tmpRunNum not in runDsMap:
                    runDsMap[tmpRunNum] = set()
                runDsMap[tmpRunNum].add(dsName)
            except:
                tmpLog.debug("failed to get metadata")
    # make string
    datasets = ''
    filesStr = []
    for tmpRunNum in runLumiMap.keys():
        for dsName in runDsMap[tmpRunNum]:
            # get files in the dataset
            tmpFilesStr = []
            tmpLFNList = []
            scope,dsn = extract_scope(dsName)
            for x in rucioclient.list_files(scope, dsn, long=True):
                tmpLFN = str(x['name'])
                LBstart_LFN = x['lumiblocknr']
                if LBstart_LFN is not None:
                    LBend_LFN = LBstart_LFN
                else:
                    # extract LBs from LFN
                    tmpItems = tmpLFN.split('.')
                    # short format
                    if len(tmpItems) < 7:
                        tmpFilesStr.append(tmpLFN)
                        continue
                    tmpLBmatch = re.search('_lb(\d+)-lb(\d+)',tmpLFN)
                    # _lbXXX-lbYYY found
                    if tmpLBmatch is not None:
                        LBstart_LFN = long(tmpLBmatch.group(1))
                        LBend_LFN   = long(tmpLBmatch.group(2))
                    else:
                        # try ._lbXYZ.
                        tmpLBmatch = re.search('\._lb(\d+)\.',tmpLFN)
                        if tmpLBmatch is not None:
                            LBstart_LFN = long(tmpLBmatch.group(1))
                            LBend_LFN   = LBstart_LFN
                        else:
                            tmpFilesStr.append(tmpLFN)
                            continue
                # check range
                inRange = False
                for LBstartXML,LBendXML in runLumiMap[tmpRunNum]:
                    if (LBstart_LFN >= LBstartXML and LBstart_LFN <= LBendXML) or \
                       (LBend_LFN >= LBstartXML and LBend_LFN <= LBendXML) or \
                       (LBstart_LFN >= LBstartXML and LBend_LFN <= LBendXML) or \
                       (LBstart_LFN <= LBstartXML and LBend_LFN >= LBendXML):
                        inRange = True
                        break
                if inRange:
                    tmpFilesStr.append(tmpLFN)
            # check if files are found
            if tmpFilesStr == []:
                tmpLog.warning('found no files with corresponding LBs in %s' % dsName)
            else:
                datasets += '%s,' % dsName
                filesStr += tmpFilesStr    
    datasets = datasets[:-1]
    if verbose:
        tmpLog.debug('converted to DS:%s LFN:%s' % (datasets,str(filesStr)))
    # return        
    return True,datasets,filesStr