Python client.Client类代码示例

 def listDatasets(self,datasetName,old=False):
     result = {}
     # extract scope from dataset
     scope,dsn = self.extract_scope(datasetName)
     if dsn.endswith('/'):
         dsn = dsn[:-1]
         collection = 'container'
     else:
         collection = 'dataset'
     filters = {'name': dsn}
     try:
         # get dids
         client = RucioClient()
         for name in client.list_dids(scope, filters, type=collection):
             vuid = hashlib.md5(scope + ':' + name).hexdigest()
             vuid = '%s-%s-%s-%s-%s' % (vuid[0:8], vuid[8:12], vuid[12:16], vuid[16:20], vuid[20:32])
             duid = vuid
             # add /
             if datasetName.endswith('/') and not name.endswith('/'):
                 name += '/'
             if old or not ':' in datasetName:
                 keyName = name
             else:
                 keyName = str('%s:%s' % (scope, name))
             if keyName not in result:
                 result[keyName] = {'duid': duid, 'vuids': [vuid]}
         return result,''
     except:
         errType,errVale = sys.exc_info()[:2]
         return None,'%s %s' % (errType,errVale)

 def registerZipFiles(self,zipMap):
     # no zip files
     if len(zipMap) == 0:
         return
     client = RucioClient()
     # loop over all zip files
     for zipFileName, zipFileAttr in zipMap.iteritems():
         # convert file attribute
         zipFile = self.convFileAttr(zipFileAttr, zipFileAttr['scope'])
         # loop over all contents
         files = []
         for conFileAttr in zipFileAttr['files']:
             # get scope
             scope,dsn = self.extract_scope(conFileAttr['ds'])
             # convert file attribute
             conFile = self.convFileAttr(conFileAttr, scope)
             conFile['type'] = 'FILE'
             if 'pfn' in conFile:
                 del conFile['pfn']
             # append files
             files.append(conFile)
         # register zip file
         for rse in zipFileAttr['rse']:
             client.add_replicas(rse=rse, files=[zipFile])
         # add files
         nFiles = 100
         iFiles = 0
         while iFiles < len(files):
             client.add_files_to_archive(scope=zipFile['scope'],
                                         name=zipFile['name'],
                                         files=files[iFiles:iFiles+nFiles])
             iFiles += nFiles

 def listFileReplicas(self,scopes,lfns,rses=None):
     try:
         client = RucioClient()
         dids = []
         iGUID = 0
         nGUID = 1000
         retVal = {}
         for scope,lfn in zip(scopes,lfns):
             iGUID += 1
             dids.append({'scope':scope,'name':lfn})
             if len(dids) % nGUID == 0 or iGUID == len(lfns):
                 for tmpDict in client.list_replicas(dids):
                     tmpLFN = str(tmpDict['name'])
                     tmpRses = tmpDict['rses'].keys()
                     # RSE selection
                     if rses is not None:
                         newRSEs = []
                         for tmpRse in tmpRses:
                             if tmpRse in rses:
                                 newRSEs.append(tmpRse)
                         tmpRses = newRSEs
                     if len(tmpRses) > 0:
                         retVal[tmpLFN] = tmpRses
                 dids = []
         return True,retVal
     except:
         errType,errVale = sys.exc_info()[:2]
         return False,'%s %s' % (errType,errVale)

 def listDatasetsByGUIDs(self,guids):
     client = RucioClient()
     result = {}
     for guid in guids:
         datasets = [str('%s:%s' % (i['scope'], i['name'])) for i in client.get_dataset_by_guid(guid)]
         result[guid] = datasets
     return result

    def resolve_replicas(self, files, protocols):

        # build list of local ddmendpoints: group by site
        # load ALL ddmconf
        self.ddmconf.update(self.si.resolveDDMConf([]))
        ddms = {}
        for ddm, dat in self.ddmconf.iteritems():
            ddms.setdefault(dat['site'], []).append(dat)

        for fdat in files:

            # build and order list of local ddms
            ddmdat = self.ddmconf.get(fdat.ddmendpoint)
            if not ddmdat:
                raise Exception("Failed to resolve ddmendpoint by name=%s send by Panda job, please check configuration. fdat=%s" % (fdat.ddmendpoint, fdat))
            if not ddmdat['site']:
                raise Exception("Failed to resolve site name of ddmendpoint=%s. please check ddm declaration: ddmconf=%s ... fdat=%s" % (fdat.ddmendpoint, ddmconf, fdat))
            localddms = ddms.get(ddmdat['site'])
            # sort/filter ddms (as possible input source)
            fdat.inputddms = self._prepare_input_ddm(ddmdat, localddms)

        # load replicats from Rucio
        from rucio.client import Client
        c = Client()

        dids = [dict(scope=e.scope, name=e.lfn) for e in files]
        schemes = ['srm', 'root', 'https', 'gsiftp']

        # Get the replica list
        try:
            replicas = c.list_replicas(dids, schemes=schemes)
        except Exception, e:
            raise PilotException("Failed to get replicas from Rucio: %s" % e, code=PilotErrors.ERR_FAILEDLFCGETREPS)

 def getRseUsage(self,rse,src='srm'):
     retMap = {}
     try:
         client = RucioClient()
         itr = client.get_rse_usage(rse)
         # look for srm
         for item in itr:
             if item['source'] == src:
                 try:
                     total = item['total']/1024/1024/1024
                 except:
                     total = None
                 try:
                     used = item['used']/1024/1024/1024
                 except:
                     used = None
                 try:
                     free = item['free']/1024/1024/1024
                 except:
                     free = None
                 retMap = {'total':total,
                           'used':used,
                           'free':free}
                 break
     except:
         pass
     return retMap

 def getZipFiles(self, dids, rses):
     try:
         client = RucioClient()
         data = []
         iGUID = 0
         nGUID = 1000
         retVal = {}
         for did in dids:
             iGUID += 1
             scope, lfn = did.split(':')
             data.append({'scope':scope,'name':lfn})
             if len(data) % nGUID == 0 or iGUID == len(dids):
                 for tmpDict in client.list_replicas(data):
                     tmpScope = str(tmpDict['scope'])
                     tmpLFN = str(tmpDict['name'])
                     tmpDID = '{0}:{1}'.format(tmpScope, tmpLFN)
                     tmpRses = tmpDict['rses'].keys()
                     # RSE selection
                     for pfn, pfnData in tmpDict['pfns'].iteritems():
                         if (rses is None or pfnData['rse'] in rses) and pfnData['domain'] == 'zip':
                             zipFileName = pfn.split('/')[-1]
                             zipFileName = re.sub('\?.+$', '', zipFileName)
                             retVal[tmpDID] = client.get_metadata(tmpScope, zipFileName)
                             break
                 data = []
         return True, retVal
     except:
         errType, errVale = sys.exc_info()[:2]
         return False, '%s %s' % (errType,errVale)

 def listFilesInDataset(self,datasetName,long=False,fileList=None):
     # extract scope from dataset
     scope,dsn = self.extract_scope(datasetName)
     if dsn.endswith('/'):
         dsn = dsn[:-1]
     client = RucioClient()
     return_dict = {}
     for x in client.list_files(scope, dsn, long=long):
         tmpLFN = str(x['name'])
         if fileList != None:
             genLFN = re.sub('\.\d+$','',tmpLFN)
             if not tmpLFN in fileList and not genLFN in fileList:
                 continue
         dq2attrs = {}
         dq2attrs['chksum'] = "ad:" + str(x['adler32'])
         dq2attrs['md5sum'] = dq2attrs['chksum']
         dq2attrs['checksum'] = dq2attrs['chksum']
         dq2attrs['fsize'] = x['bytes']
         dq2attrs['filesize'] = dq2attrs['fsize']
         dq2attrs['scope'] = str(x['scope'])
         dq2attrs['events'] = str(x['events'])
         if long:
             dq2attrs['lumiblocknr'] = str(x['lumiblocknr'])
         guid = str('%s-%s-%s-%s-%s' % (x['guid'][0:8], x['guid'][8:12], x['guid'][12:16], x['guid'][16:20], x['guid'][20:32]))
         dq2attrs['guid'] = guid
         return_dict[tmpLFN] = dq2attrs
     return (return_dict, None)

 def registerDatasetLocation(self,dsn,rses,lifetime=None,owner=None):
     if lifetime != None:
         lifetime = lifetime*24*60*60
     scope,dsn = self.extract_scope(dsn)    
     dids = []
     did = {'scope': scope, 'name': dsn}
     dids.append(did)
     # make location
     rses.sort()
     location = '|'.join(rses)
     # check if a replication rule already exists
     client = RucioClient()
     # owner
     if owner == None:
         owner = client.account
     for rule in client.list_did_rules(scope=scope, name=dsn):
         if (rule['rse_expression'] == location) and (rule['account'] == client.account):
             return True
     try:
         client.add_replication_rule(dids=dids,copies=1,rse_expression=location,weight=None,
                                     lifetime=lifetime, grouping='DATASET', account=owner,
                                     locked=False, notify='N',ignore_availability=True)
     except Duplicate:
         pass
     return True

 def deleteFilesFromDataset(self,datasetName,files):
     # extract scope from dataset
     scope,dsn = self.extract_scope(datasetName)
     client = RucioClient()
     try:
         # delete files
         client.detach_dids(scope=scope,name=dsn,dids=files)
     except DataIdentifierNotFound:
         pass

 def closeDataset(self,dsn):
     # register dataset
     client = RucioClient()
     try:
         scope,dsn = self.extract_scope(dsn)
         client.set_status(scope,dsn,open=False)
     except (UnsupportedOperation,DataIdentifierNotFound):
         pass
     return True

 def getMetaData(self,dsn):
     # register dataset
     client = RucioClient()
     try:
         scope,dsn = self.extract_scope(dsn)
         return True,client.get_metadata(scope,dsn)
     except DataIdentifierNotFound:
         return True,None
     except:
         errType,errVale = sys.exc_info()[:2]
         return False,'%s %s' % (errType,errVale)

 def setMetaData(self,dsn,metadata=None):
     # register dataset
     client = RucioClient()
     try:
         scope,dsn = self.extract_scope(dsn)
         for tmpKey,tmpValue in metadata.iteritems():
             client.set_metadata(scope,dsn,key=tmpKey,value=tmpValue)
     except:
         errType,errVale = sys.exc_info()[:2]
         return False,'%s %s' % (errType,errVale)
     return True,''

 def registerFiles(self,files,rse):
     client = RucioClient()
     try:
         # add replicas
         client.add_replicas(files=files,rse=rse)
     except FileAlreadyExists:
         pass
     try:
         # add rule
         client.add_replication_rule(files,copies=1,rse_expression=rse)
     except DuplicateRule:
         pass

 def getDatasetSize(self,datasetName):
     # extract scope from dataset
     scope,dsn = self.extract_scope(datasetName)
     client = RucioClient()
     tSize = 0
     try:
         for x in client.list_files(scope, dsn, long=long):
             tSize += x['bytes']
         return True,tSize
     except DataIdentifierNotFound:
         return None,'dataset not found'
     except:
         errtype, errvalue = sys.exc_info()[:2]
         errMsg = '{0} {1}'.format(errtype.__name__, errvalue)
         return False,errMsg