Python Client.get_metadata方法代碼示例

# 需要導入模塊: from rucio.client import Client [as 別名]
# 或者: from rucio.client.Client import get_metadata [as 別名]
 def getZipFiles(self, dids, rses):
     try:
         client = RucioClient()
         data = []
         iGUID = 0
         nGUID = 1000
         retVal = {}
         for did in dids:
             iGUID += 1
             scope, lfn = did.split(':')
             data.append({'scope':scope,'name':lfn})
             if len(data) % nGUID == 0 or iGUID == len(dids):
                 for tmpDict in client.list_replicas(data):
                     tmpScope = str(tmpDict['scope'])
                     tmpLFN = str(tmpDict['name'])
                     tmpDID = '{0}:{1}'.format(tmpScope, tmpLFN)
                     tmpRses = tmpDict['rses'].keys()
                     # RSE selection
                     for pfn, pfnData in tmpDict['pfns'].iteritems():
                         if (rses is None or pfnData['rse'] in rses) and pfnData['domain'] == 'zip':
                             zipFileName = pfn.split('/')[-1]
                             zipFileName = re.sub('\?.+$', '', zipFileName)
                             retVal[tmpDID] = client.get_metadata(tmpScope, zipFileName)
                             break
                 data = []
         return True, retVal
     except:
         errType, errVale = sys.exc_info()[:2]
         return False, '%s %s' % (errType,errVale)

# 需要導入模塊: from rucio.client import Client [as 別名]
# 或者: from rucio.client.Client import get_metadata [as 別名]
 def getMetaData(self,dsn):
     # register dataset
     client = RucioClient()
     try:
         scope,dsn = self.extract_scope(dsn)
         return True,client.get_metadata(scope,dsn)
     except DataIdentifierNotFound:
         return True,None
     except:
         errType,errVale = sys.exc_info()[:2]
         return False,'%s %s' % (errType,errVale)

# 需要導入模塊: from rucio.client import Client [as 別名]
# 或者: from rucio.client.Client import get_metadata [as 別名]
def convertGoodRunListXMLtoDS(tmpLog,goodRunListXML,goodRunDataType='',goodRunProdStep='',
                              goodRunListDS='',verbose=False):
    tmpLog.info('trying to convert GoodRunListXML to a list of datasets')  
    # return for failure
    failedRet = False,'',[]
    # import pyAMI
    try:
        import pyAMI.utils
        import pyAMI.client
        import pyAMI.atlas.api as AtlasAPI
    except:
        errType,errValue = sys.exc_info()[:2]
        print "%s %s" % (errType,errValue)
        tmpLog.error('cannot import pyAMI module')
        return failedRet
    # read XML
    try:
        gl_xml = open(goodRunListXML)
    except:
        tmpLog.error('cannot open %s' % goodRunListXML)
        return failedRet
    # parse XML to get run/lumi
    runLumiMap = {}
    import xml.dom.minidom
    rootDOM = xml.dom.minidom.parse(goodRunListXML)
    for tmpLumiBlock in rootDOM.getElementsByTagName('LumiBlockCollection'):
        for tmpRunNode in tmpLumiBlock.getElementsByTagName('Run'):
            tmpRunNum  = long(tmpRunNode.firstChild.data)
            for tmpLBRange in tmpLumiBlock.getElementsByTagName('LBRange'):
                tmpLBStart = long(tmpLBRange.getAttribute('Start'))
                tmpLBEnd   = long(tmpLBRange.getAttribute('End'))
                # append
                if not runLumiMap.has_key(tmpRunNum):
                    runLumiMap[tmpRunNum] = []
                runLumiMap[tmpRunNum].append((tmpLBStart,tmpLBEnd))
        kwargs = dict()
    kwargs['run_number'] = [unicode(x) for x in runLumiMap.keys()]
    kwargs['ami_status'] = 'VALID'
    if goodRunDataType != '':
        kwargs['type'] = goodRunDataType
    if goodRunProdStep != '':    
        kwargs['stream'] = goodRunProdStep
    if verbose:
        tmpLog.debug(kwargs)
    # convert for wildcard
    goodRunListDS = goodRunListDS.replace('*','.*')
    # list of datasets
    if goodRunListDS == '':
        goodRunListDS = []
    else:
        goodRunListDS = goodRunListDS.split(',')
    # execute
    try:
        amiclient = pyAMI.client.Client('atlas')
        amiOutDict = pyAMI.utils.smart_execute(amiclient, 'datasets', [], None, None, None, False, **kwargs).get_rows()
    except:
        errType,errValue = sys.exc_info()[:2]
        tmpLog.error("%s %s" % (errType,errValue))
        tmpLog.error('pyAMI failed')
        return failedRet
    # get dataset map
    if verbose:
        tmpLog.debug(amiOutDict)
    # parse
    rucioclient = RucioClient()
    datasetListFromAMI = []
    runDsMap = dict()
    for tmpVal in amiOutDict:
        if tmpVal.has_key('ldn'):
            dsName = str(tmpVal['ldn'])
            # check dataset names
            if goodRunListDS == []:    
                matchFlag = True
            else:
                matchFlag = False
                for tmpPatt in goodRunListDS:
                    if re.search(tmpPatt,dsName) != None:
                        matchFlag = True
            if not matchFlag:
                continue
            # get metadata
            try:
                tmpLog.debug("getting metadata for %s" % dsName)
                scope,dsn = extract_scope(dsName)
                meta =  rucioclient.get_metadata(scope, dsn)
                if meta['did_type'] == 'COLLECTION':
                    dsName += '/'
                datasetListFromAMI.append(dsName)
                tmpRunNum = meta['run_number']
                tmpLog.debug("run number : %s" % tmpRunNum)
                if tmpRunNum not in runDsMap:
                    runDsMap[tmpRunNum] = set()
                runDsMap[tmpRunNum].add(dsName)
            except:
                tmpLog.debug("failed to get metadata")
    # make string
    datasets = ''
    filesStr = []
    for tmpRunNum in runLumiMap.keys():
        for dsName in runDsMap[tmpRunNum]:
#.........這裏部分代碼省略.........