本文整理匯總了Python中rucio.client.Client.add_files_to_datasets方法的典型用法代碼示例。如果您正苦於以下問題：Python Client.add_files_to_datasets方法的具體用法？Python Client.add_files_to_datasets怎麽用？Python Client.add_files_to_datasets使用的例子？那麽, 這裏精選的方法代碼示例或許可以為您提供幫助。您也可以進一步了解該方法所在類rucio.client.Client的用法示例。
在下文中一共展示了Client.add_files_to_datasets方法的4個代碼示例，這些例子默認根據受歡迎程度排序。您可以為喜歡或者感覺有用的代碼點讚，您的評價將有助於係統推薦出更棒的Python代碼示例。
示例1: registerFilesInDataset
# 需要導入模塊: from rucio.client import Client [as 別名]
# 或者: from rucio.client.Client import add_files_to_datasets [as 別名]
def registerFilesInDataset(self,idMap,filesWoRSEs=None):
# loop over all rse
attachmentList = []
for rse,tmpMap in idMap.iteritems():
# loop over all datasets
for datasetName,fileList in tmpMap.iteritems():
# extract scope from dataset
scope,dsn = self.extract_scope(datasetName)
filesWithRSE = []
filesWoRSE = []
for tmpFile in fileList:
# convert file attribute
file = self.convFileAttr(tmpFile, scope)
# append files
if rse != None and (filesWoRSEs is None or file['name'] not in filesWoRSEs):
filesWithRSE.append(file)
else:
if 'pfn' in file:
del file['pfn']
filesWoRSE.append(file)
# add attachment
if len(filesWithRSE) > 0:
nFiles = 100
iFiles = 0
while iFiles < len(filesWithRSE):
attachment = {'scope':scope,
'name':dsn,
'dids':filesWithRSE[iFiles:iFiles+nFiles],
'rse':rse}
attachmentList.append(attachment)
iFiles += nFiles
if len(filesWoRSE) > 0:
nFiles = 100
iFiles = 0
while iFiles < len(filesWoRSE):
attachment = {'scope':scope,
'name':dsn,
'dids':filesWoRSE[iFiles:iFiles+nFiles]}
attachmentList.append(attachment)
iFiles += nFiles
# add files
client = RucioClient()
client.add_files_to_datasets(attachmentList, ignore_duplicate=True)
return True示例2: registerFilesInDataset
# 需要導入模塊: from rucio.client import Client [as 別名]
# 或者: from rucio.client.Client import add_files_to_datasets [as 別名]
def registerFilesInDataset(self,idMap):
# loop over all rse
attachmentList = []
for rse,tmpMap in idMap.iteritems():
# loop over all datasets
for datasetName,fileList in tmpMap.iteritems():
# extract scope from dataset
scope,dsn = self.extract_scope(datasetName)
files = []
for tmpFile in fileList:
# extract scope from LFN if available
lfn = tmpFile['lfn']
if ':' in lfn:
s, lfn = lfn.split(':')
else:
s = scope
# set metadata
meta = {'guid': tmpFile['guid']}
if 'events' in tmpFile:
meta['events'] = tmpFile['events']
if 'lumiblocknr' in tmpFile:
meta['lumiblocknr'] = tmpFile['lumiblocknr']
# set mandatory fields
file = {'scope': s,
'name' : lfn,
'bytes': tmpFile['size'],
'meta' : meta}
checksum = tmpFile['checksum']
if checksum.startswith('md5:'):
file['md5'] = checksum[4:]
elif checksum.startswith('ad:'):
file['adler32'] = checksum[3:]
if 'surl' in tmpFile:
file['pfn'] = tmpFile['surl']
# append files
files.append(file)
# add attachment
attachment = {'scope':scope,
'name':dsn,
'dids':files}
if rse != None:
attachment['rse'] = rse
attachmentList.append(attachment)
# add files
client = RucioClient()
return client.add_files_to_datasets(attachmentList,ignore_duplicate=True)示例3: trigger_stage_out
# 需要導入模塊: from rucio.client import Client [as 別名]
# 或者: from rucio.client.Client import add_files_to_datasets [as 別名]
#.........這裏部分代碼省略.........
return None,errStr
# print out the file list
tmpLog.debug('fileList - {0}'.format(fileList))
# create the dataset and add files to it and create a transfer rule
try:
# register dataset
rucioAPI = RucioClient()
tmpLog.debug('register {0}:{1} rse = {2} meta=(hidden: True) lifetime = {3}'
.format(datasetScope, datasetName,srcRSE,(30*24*60*60)))
try:
rucioAPI.add_dataset(datasetScope, datasetName,
meta={'hidden': True},
lifetime=30 * 24 * 60 * 60,
rse=srcRSE
)
except DataIdentifierAlreadyExists:
# ignore even if the dataset already exists
pass
except Exception:
errMsg = 'Could not create dataset {0}:{1} srcRSE - {2}'.format(datasetScope,
datasetName,
srcRSE)
core_utils.dump_error_message(tmpLog)
tmpLog.error(errMsg)
return None,errMsg
# add files to dataset
# add 500 files at a time
numfiles = len(fileList)
maxfiles = 500
numslices = numfiles/maxfiles
if (numfiles%maxfiles) > 0 :
numslices = numslices + 1
start = 0
for i in range(numslices) :
try:
stop = start + maxfiles
if stop > numfiles :
stop = numfiles
rucioAPI.add_files_to_datasets([{'scope': datasetScope,
'name': datasetName,
'dids': fileList[start:stop],
'rse': srcRSE}],
ignore_duplicate=True)
start = stop
except FileAlreadyExists:
# ignore if files already exist
pass
except Exception:
errMsg = 'Could not add files to DS - {0}:{1} rse - {2} files - {3}'.format(datasetScope,
datasetName,
srcRSE,
fileList)
core_utils.dump_error_message(tmpLog)
tmpLog.error(errMsg)
return None,errMsg
# add rule
try:
tmpDID = dict()
tmpDID['scope'] = datasetScope
tmpDID['name'] = datasetName
tmpRet = rucioAPI.add_replication_rule([tmpDID], 1, dstRSE,
lifetime=30 * 24 * 60 * 60)
ruleIDs = tmpRet[0]
tmpLog.debug('registered dataset {0}:{1} with rule {2}'.format(datasetScope, datasetName,
str(ruleIDs)))
# group the output files together by the Rucio transfer rule
jobspec.set_groups_to_files({ruleIDs: {'lfns': lfns,'groupStatus': 'pending'}})
msgStr = 'jobspec.set_groups_to_files -Rucio rule - {0}, lfns - {1}, groupStatus - pending'.format(ruleIDs,lfns)
tmpLog.debug(msgStr)
tmpLog.debug('call self.dbInterface.set_file_group(jobspec.get_output_file_specs(skip_done=True),ruleIDs,pending)')
tmpStat = self.dbInterface.set_file_group(jobspec.get_output_file_specs(skip_done=True),ruleIDs,'transferring')
tmpLog.debug('called self.dbInterface.set_file_group(jobspec.get_output_file_specs(skip_done=True),ruleIDs,transferring)')
tmpStat = True
tmpMsg = 'created Rucio rule successfully'
except DuplicateRule:
# ignore duplicated rule
tmpLog.debug('rule is already available')
except Exception:
errMsg = 'Error creating rule for dataset {0}:{1}'.format(datasetScope, datasetName)
core_utils.dump_error_message(tmpLog)
tmpLog.debug(errMsg)
return None,errMsg
# update file group status
self.dbInterface.update_file_group_status(ruleIDs, 'transferring')
except Exception:
core_utils.dump_error_message(tmpLog)
# treat as a temporary error
tmpStat = None
tmpMsg = 'failed to add a rule for {0}:{1}'.format(datasetScope, datasetName)
# Now test for any errors
if errors:
for error in errors:
tmpLog.debug('copy error source {0} destination {1} Reason {2}'.format(error[0],error[1],error[2]))
raise Error(errors)
# otherwise we are OK
tmpLog.debug('stop')
return tmpStat,tmpMsg示例4: trigger_stage_out
# 需要導入模塊: from rucio.client import Client [as 別名]
# 或者: from rucio.client.Client import add_files_to_datasets [as 別名]
def trigger_stage_out(self, jobspec):
# make logger
tmpLog = self.make_logger(_logger, 'PandaID={0} ThreadID={1} '.format(jobspec.PandaID,
threading.current_thread().ident),
method_name='trigger_stage_out')
tmpLog.debug('executing base trigger_stage_out')
tmpStat, tmpMsg = YodaRseDirect.trigger_stage_out(self, jobspec)
tmpLog.debug('got {0} {1}'.format(tmpStat, tmpMsg))
if tmpStat is not True:
return tmpStat, tmpMsg
# Now that output files have been all copied to Local RSE register transient dataset
# loop over all transfers
tmpStat = None
tmpMsg = ''
srcRSE = None
dstRSE = None
datasetName = 'panda.harvester.{0}.{1}'.format(jobspec.PandaID,str(uuid.uuid4()))
datasetScope = 'transient'
# get destination endpoint
nucleus = jobspec.jobParams['nucleus']
agis = self.dbInterface.get_cache('panda_queues.json').data
dstRSE = [agis[x]["astorages"]['pr'][0] for x in agis if agis[x]["atlas_site"] == nucleus][0]
# get the list of output files to transfer
fileSpecs = jobspec.get_output_file_specs(skip_done=True)
fileList = []
lfns = []
for fileSpec in fileSpecs:
tmpFile = dict()
tmpFile['scope'] = datasetScope
tmpFile['name'] = fileSpec.lfn
tmpFile['bytes'] = fileSpec.fsize
tmpFile['adler32'] = fileSpec.chksum
tmpFile['meta'] = {'guid': fileSpec.fileAttributes['guid']}
fileList.append(tmpFile)
lfns.append(fileSpec.lfn)
# get source RSE
if srcRSE is None and fileSpec.objstoreID is not None:
ddm = self.dbInterface.get_cache('agis_ddmendpoints.json').data
srcRSE = [x for x in ddm if ddm[x]["id"] == fileSpec.objstoreID][0]
# test that srcRSE and dstRSE are defined
errStr = ''
if srcRSE is None:
errStr = 'Source RSE is not defined '
if dstRSE is None:
errStr = errStr + ' Desitination RSE is not defined'
if (srcRSE is None) or (dstRSE is None) :
tmpLog.error(errStr)
return False,errStr
# create the dataset and add files to it and create a transfer rule
try:
# register dataset
tmpLog.debug('register {0}:{1}'.format(datasetScope, datasetName))
rucioAPI = RucioClient()
try:
rucioAPI.add_dataset(datasetScope, datasetName,
meta={'hidden': True},
lifetime=30 * 24 * 60 * 60,
rse=srcRSE
)
except DataIdentifierAlreadyExists:
# ignore even if the dataset already exists
pass
except Exception:
tmpLog.error('Could not create dataset with scope: {0} Name: {1} in Rucio'
.format(datasetScope,datasetName))
raise
# add files to dataset
try:
rucioAPI.add_files_to_datasets([{'scope': datasetScope,
'name': datasetName,
'dids': fileList,
'rse': srcRSE}],
ignore_duplicate=True)
except FileAlreadyExists:
# ignore if files already exist
pass
except Exception:
tmpLog.error('Could add files to dataset with scope: {0} Name: {1} in Rucio'
.format(datasetScope,datasetName))
raise
# add rule
try:
tmpDID = dict()
tmpDID['scope'] = datasetScope
tmpDID['name'] = datasetName
tmpRet = rucioAPI.add_replication_rule([tmpDID], 1, dstRSE,
lifetime=30 * 24 * 60 * 60)
ruleIDs = tmpRet[0]
tmpLog.debug('registered dataset {0}:{1} with rule {2}'.format(datasetScope, datasetName,
str(ruleIDs)))
# group the output files together by the Rucio transfer rule
jobspec.set_groups_to_files({ruleIDs: {'lfns': lfns,'groupStatus': 'pending'}})
msgStr = 'jobspec.set_groups_to_files -Rucio rule - {0}, lfns - {1}, groupStatus - pending'.format(ruleIDs,lfns)
tmpLog.debug(msgStr)
#.........這裏部分代碼省略.........