Python Client.add_files_to_dataset方法代码示例

# 需要导入模块: from rucio.client import Client [as 别名]
# 或者: from rucio.client.Client import add_files_to_dataset [as 别名]
 def registerDataset(self,dsn,lfns=[],guids=[],sizes=[],checksums=[],lifetime=None,scope=None,metadata=None):
     presetScope = scope
     files = []
     for lfn, guid, size, checksum in zip(lfns, guids, sizes, checksums):
         if lfn.find(':') > -1:
             s, lfn = lfn.split(':')[0], lfn.split(':')[1]
         else:
             s = scope
         file = {'scope': s, 'name': lfn, 'bytes': size, 'meta': {'guid': guid}}
         if checksum.startswith('md5:'):
             file['md5'] = checksum[4:]
         elif checksum.startswith('ad:'):
             file['adler32'] = checksum[3:]
         files.append(file)
     # register dataset
     client = RucioClient()
     try:
         scope,dsn = self.extract_scope(dsn)
         if presetScope is not None:
             scope = presetScope
         client.add_dataset(scope=scope, name=dsn, meta=metadata)
         if lifetime != None:
             client.set_metadata(scope,dsn,key='lifetime',value=lifetime*86400)
     except DataIdentifierAlreadyExists:
         pass
     # open dataset just in case
     try:
         client.set_status(scope,dsn,open=True)
     except:
         pass
     # add files
     if len(files) > 0:
         iFiles = 0
         nFiles = 1000
         while iFiles < len(files):
             tmpFiles = files[iFiles:iFiles+nFiles]
             try:
                 client.add_files_to_dataset(scope=scope,name=dsn,files=tmpFiles, rse=None)
             except FileAlreadyExists:
                 for f in tmpFiles:
                     try:
                         client.add_files_to_dataset(scope=scope, name=dsn, files=[f], rse=None)
                     except FileAlreadyExists:
                         pass
             iFiles += nFiles
     vuid = hashlib.md5(scope+':'+dsn).hexdigest()
     vuid = '%s-%s-%s-%s-%s' % (vuid[0:8], vuid[8:12], vuid[12:16], vuid[16:20], vuid[20:32])
     duid = vuid
     return {'duid': duid, 'version': 1, 'vuid': vuid}

# 需要导入模块: from rucio.client import Client [as 别名]
# 或者: from rucio.client.Client import add_files_to_dataset [as 别名]
 def trigger_stage_out(self, jobspec):
     # make logger
     tmpLog = self.make_logger(baseLogger, 'PandaID={0}'.format(jobspec.PandaID),
                               method_name='trigger_stage_out')
     tmpLog.debug('start')
     # loop over all files
     files = dict()
     transferIDs = dict()
     transferDatasets = dict()
     fileAttrs = jobspec.get_output_file_attributes()
     for fileSpec in jobspec.outFiles:
         # skip zipped files
         if fileSpec.zipFileID is not None:
             continue
         # skip if already processed
         if 'transferDataset' in fileSpec.fileAttributes:
             if fileSpec.fileType not in transferDatasets:
                 transferDatasets[fileSpec.fileType] = fileSpec.fileAttributes['transferDataset']
             if fileSpec.fileType not in transferIDs:
                 transferIDs[fileSpec.fileType] = fileSpec.fileAttributes['transferID']
             continue
         # set OS ID
         if fileSpec.fileType == ['es_output', 'zip_output']:
             fileSpec.objstoreID = self.objStoreID_ES
         # make path where file is copied for transfer
         if fileSpec.fileType != 'zip_output':
             scope = fileAttrs[fileSpec.lfn]['scope']
             datasetName = fileAttrs[fileSpec.lfn]['dataset']
         else:
             # use panda scope for zipped files
             scope = self.scopeForTmp
             datasetName = 'dummy'
         srcPath = fileSpec.path
         dstPath = mover_utils.construct_file_path(self.srcBasePath, scope, fileSpec.lfn)
         # remove
         if os.path.exists(dstPath):
             os.remove(dstPath)
         # copy
         tmpLog.debug('copy src={srcPath} dst={dstPath}'.format(srcPath=srcPath, dstPath=dstPath))
         dstDir = os.path.dirname(dstPath)
         if not os.path.exists(dstDir):
             os.makedirs(dstDir)
         shutil.copyfile(srcPath, dstPath)
         # collect files
         tmpFile = dict()
         tmpFile['scope'] = scope
         tmpFile['name'] = fileSpec.lfn
         tmpFile['bytes'] = fileSpec.fsize
         if fileSpec.fileType not in files:
             files[fileSpec.fileType] = []
         files[fileSpec.fileType].append(tmpFile)
     # loop over all file types to be registered to rucio
     rucioAPI = RucioClient()
     for fileType, fileList in iteritems(files):
         # set destination RSE
         if fileType in ['es_output', 'zip_output']:
             dstRSE = self.dstRSE_ES
         elif fileType == 'output':
             dstRSE = self.dstRSE_Out
         elif fileType == 'log':
             dstRSE = self.dstRSE_Log
         else:
             errMsg = 'unsupported file type {0}'.format(fileType)
             tmpLog.error(errMsg)
             return (False, errMsg)
         # skip if destination is None
         if dstRSE is None:
             continue
         # make datasets if missing
         if fileType not in transferDatasets:
             try:
                 tmpScope = self.scopeForTmp
                 tmpDS = 'panda.harvester_stage_out.{0}'.format(str(uuid.uuid4()))
                 rucioAPI.add_dataset(tmpScope, tmpDS,
                                      meta={'hidden': True},
                                      lifetime=30*24*60*60,
                                      files=fileList,
                                      rse=self.srcRSE
                                      )
                 transferDatasets[fileType] = tmpDS
                 # add rule
                 tmpDID = dict()
                 tmpDID['scope'] = tmpScope
                 tmpDID['name'] = tmpDS
                 tmpRet = rucioAPI.add_replication_rule([tmpDID], 1, dstRSE,
                                                        lifetime=30*24*60*60
                                                        )
                 tmpTransferIDs = tmpRet[0]
                 transferIDs[fileType] = tmpTransferIDs
                 tmpLog.debug('register dataset {0} with rule {1}'.format(tmpDS, str(tmpTransferIDs)))
             except:
                 errMsg = core_utils.dump_error_message(tmpLog)
                 return (False, errMsg)
         else:
             # add files to existing dataset
             try:
                 tmpScope = self.scopeForTmp
                 tmpDS = transferDatasets[fileType]
                 rucioAPI.add_files_to_dataset(tmpScope, tmpDS, fileList, self.srcRSE)
                 tmpLog.debug('added files to {0}'.format(tmpDS))
#.........这里部分代码省略.........