本文整理汇总了Python中rucio.client.Client.add_dataset方法的典型用法代码示例。如果您正苦于以下问题：Python Client.add_dataset方法的具体用法？Python Client.add_dataset怎么用？Python Client.add_dataset使用的例子？那么, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类rucio.client.Client的用法示例。
在下文中一共展示了Client.add_dataset方法的5个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: registerDataset
# 需要导入模块: from rucio.client import Client [as 别名]
# 或者: from rucio.client.Client import add_dataset [as 别名]
def registerDataset(self,dsn,lfns=[],guids=[],sizes=[],checksums=[],lifetime=None,scope=None,metadata=None):
presetScope = scope
files = []
for lfn, guid, size, checksum in zip(lfns, guids, sizes, checksums):
if lfn.find(':') > -1:
s, lfn = lfn.split(':')[0], lfn.split(':')[1]
else:
s = scope
file = {'scope': s, 'name': lfn, 'bytes': size, 'meta': {'guid': guid}}
if checksum.startswith('md5:'):
file['md5'] = checksum[4:]
elif checksum.startswith('ad:'):
file['adler32'] = checksum[3:]
files.append(file)
# register dataset
client = RucioClient()
try:
scope,dsn = self.extract_scope(dsn)
if presetScope is not None:
scope = presetScope
client.add_dataset(scope=scope, name=dsn, meta=metadata)
if lifetime != None:
client.set_metadata(scope,dsn,key='lifetime',value=lifetime*86400)
except DataIdentifierAlreadyExists:
pass
# open dataset just in case
try:
client.set_status(scope,dsn,open=True)
except:
pass
# add files
if len(files) > 0:
iFiles = 0
nFiles = 1000
while iFiles < len(files):
tmpFiles = files[iFiles:iFiles+nFiles]
try:
client.add_files_to_dataset(scope=scope,name=dsn,files=tmpFiles, rse=None)
except FileAlreadyExists:
for f in tmpFiles:
try:
client.add_files_to_dataset(scope=scope, name=dsn, files=[f], rse=None)
except FileAlreadyExists:
pass
iFiles += nFiles
vuid = hashlib.md5(scope+':'+dsn).hexdigest()
vuid = '%s-%s-%s-%s-%s' % (vuid[0:8], vuid[8:12], vuid[12:16], vuid[16:20], vuid[20:32])
duid = vuid
return {'duid': duid, 'version': 1, 'vuid': vuid}示例2: trigger_stage_out
# 需要导入模块: from rucio.client import Client [as 别名]
# 或者: from rucio.client.Client import add_dataset [as 别名]
#.........这里部分代码省略.........
tmpLog.debug('srcRSE - {0} defined from agis_ddmendpoints.json'.format(srcRSE))
except (IOError, os.error) as why:
errors.append((srcURL, dstURL, str(why)))
else :
errors.append((srcURL, dstURL, 'Source file missing'))
ifile += 1
# test that srcRSE and dstRSE are defined
tmpLog.debug('srcRSE - {0} dstRSE - {1}'.format(srcRSE,dstRSE))
errStr = ''
if srcRSE is None:
errStr = 'Source RSE is not defined '
if dstRSE is None:
errStr = errStr + ' Desitination RSE is not defined'
if (srcRSE is None) or (dstRSE is None) :
tmpLog.error(errStr)
return None,errStr
# test to see if there are any files to add dataset
if len(fileList) == 0:
errStr = 'There are no files to add to database'
tmpLog.error(errStr)
return None,errStr
# print out the file list
tmpLog.debug('fileList - {0}'.format(fileList))
# create the dataset and add files to it and create a transfer rule
try:
# register dataset
rucioAPI = RucioClient()
tmpLog.debug('register {0}:{1} rse = {2} meta=(hidden: True) lifetime = {3}'
.format(datasetScope, datasetName,srcRSE,(30*24*60*60)))
try:
rucioAPI.add_dataset(datasetScope, datasetName,
meta={'hidden': True},
lifetime=30 * 24 * 60 * 60,
rse=srcRSE
)
except DataIdentifierAlreadyExists:
# ignore even if the dataset already exists
pass
except Exception:
errMsg = 'Could not create dataset {0}:{1} srcRSE - {2}'.format(datasetScope,
datasetName,
srcRSE)
core_utils.dump_error_message(tmpLog)
tmpLog.error(errMsg)
return None,errMsg
# add files to dataset
# add 500 files at a time
numfiles = len(fileList)
maxfiles = 500
numslices = numfiles/maxfiles
if (numfiles%maxfiles) > 0 :
numslices = numslices + 1
start = 0
for i in range(numslices) :
try:
stop = start + maxfiles
if stop > numfiles :
stop = numfiles
rucioAPI.add_files_to_datasets([{'scope': datasetScope,
'name': datasetName,
'dids': fileList[start:stop],
'rse': srcRSE}],示例3: RucioClient
# 需要导入模块: from rucio.client import Client [as 别名]
# 或者: from rucio.client.Client import add_dataset [as 别名]
import sys
from rucio.client import Client as RucioClient
client = RucioClient()
# scope = sys.argv[1]
# name = sys.argv[2]
# rep.list_replicas([{'scope':'ams-2011B-ISS.B620-pass4', 'name':'1340252898.00981893.root'}])
# for x in rep.list_replicas([{'scope':scope, 'name':name}]):
# print x
from rucio.common.utils import generate_uuid
account = 'chenghsi'
scope = 'ams-user-chenghsi'
name = 'file20150709T105442'
# client.add_replication_rule(dids=[{'scope': scope, 'name':name}],copies=1,rse_expression='TW-EOS02_AMS02SCRATCHDISK',weight=None,
# lifetime=1, grouping='DATASET', account=account, locked=False, notify='N',ignore_availability=True)
client.add_dataset(scope=scope, name='dataset001')
# import pdb; pdb.set_trace()
# for key, value in client.get_metadata(scope, name).iteritems():
# print key, value
# client.set_metadata(scope, name, 'guid', generate_uuid())
示例4: trigger_stage_out
# 需要导入模块: from rucio.client import Client [as 别名]
# 或者: from rucio.client.Client import add_dataset [as 别名]
def trigger_stage_out(self, jobspec):
# make logger
tmpLog = self.make_logger(baseLogger, 'PandaID={0}'.format(jobspec.PandaID),
method_name='trigger_stage_out')
tmpLog.debug('start')
# loop over all files
files = dict()
transferIDs = dict()
transferDatasets = dict()
fileAttrs = jobspec.get_output_file_attributes()
for fileSpec in jobspec.outFiles:
# skip zipped files
if fileSpec.zipFileID is not None:
continue
# skip if already processed
if 'transferDataset' in fileSpec.fileAttributes:
if fileSpec.fileType not in transferDatasets:
transferDatasets[fileSpec.fileType] = fileSpec.fileAttributes['transferDataset']
if fileSpec.fileType not in transferIDs:
transferIDs[fileSpec.fileType] = fileSpec.fileAttributes['transferID']
continue
# set OS ID
if fileSpec.fileType == ['es_output', 'zip_output']:
fileSpec.objstoreID = self.objStoreID_ES
# make path where file is copied for transfer
if fileSpec.fileType != 'zip_output':
scope = fileAttrs[fileSpec.lfn]['scope']
datasetName = fileAttrs[fileSpec.lfn]['dataset']
else:
# use panda scope for zipped files
scope = self.scopeForTmp
datasetName = 'dummy'
srcPath = fileSpec.path
dstPath = mover_utils.construct_file_path(self.srcBasePath, scope, fileSpec.lfn)
# remove
if os.path.exists(dstPath):
os.remove(dstPath)
# copy
tmpLog.debug('copy src={srcPath} dst={dstPath}'.format(srcPath=srcPath, dstPath=dstPath))
dstDir = os.path.dirname(dstPath)
if not os.path.exists(dstDir):
os.makedirs(dstDir)
shutil.copyfile(srcPath, dstPath)
# collect files
tmpFile = dict()
tmpFile['scope'] = scope
tmpFile['name'] = fileSpec.lfn
tmpFile['bytes'] = fileSpec.fsize
if fileSpec.fileType not in files:
files[fileSpec.fileType] = []
files[fileSpec.fileType].append(tmpFile)
# loop over all file types to be registered to rucio
rucioAPI = RucioClient()
for fileType, fileList in iteritems(files):
# set destination RSE
if fileType in ['es_output', 'zip_output']:
dstRSE = self.dstRSE_ES
elif fileType == 'output':
dstRSE = self.dstRSE_Out
elif fileType == 'log':
dstRSE = self.dstRSE_Log
else:
errMsg = 'unsupported file type {0}'.format(fileType)
tmpLog.error(errMsg)
return (False, errMsg)
# skip if destination is None
if dstRSE is None:
continue
# make datasets if missing
if fileType not in transferDatasets:
try:
tmpScope = self.scopeForTmp
tmpDS = 'panda.harvester_stage_out.{0}'.format(str(uuid.uuid4()))
rucioAPI.add_dataset(tmpScope, tmpDS,
meta={'hidden': True},
lifetime=30*24*60*60,
files=fileList,
rse=self.srcRSE
)
transferDatasets[fileType] = tmpDS
# add rule
tmpDID = dict()
tmpDID['scope'] = tmpScope
tmpDID['name'] = tmpDS
tmpRet = rucioAPI.add_replication_rule([tmpDID], 1, dstRSE,
lifetime=30*24*60*60
)
tmpTransferIDs = tmpRet[0]
transferIDs[fileType] = tmpTransferIDs
tmpLog.debug('register dataset {0} with rule {1}'.format(tmpDS, str(tmpTransferIDs)))
except:
errMsg = core_utils.dump_error_message(tmpLog)
return (False, errMsg)
else:
# add files to existing dataset
try:
tmpScope = self.scopeForTmp
tmpDS = transferDatasets[fileType]
rucioAPI.add_files_to_dataset(tmpScope, tmpDS, fileList, self.srcRSE)
tmpLog.debug('added files to {0}'.format(tmpDS))
#.........这里部分代码省略.........
示例5: trigger_stage_out
# 需要导入模块: from rucio.client import Client [as 别名]
# 或者: from rucio.client.Client import add_dataset [as 别名]
def trigger_stage_out(self, jobspec):
# make logger
tmpLog = self.make_logger(_logger, 'PandaID={0} ThreadID={1} '.format(jobspec.PandaID,
threading.current_thread().ident),
method_name='trigger_stage_out')
tmpLog.debug('executing base trigger_stage_out')
tmpStat, tmpMsg = YodaRseDirect.trigger_stage_out(self, jobspec)
tmpLog.debug('got {0} {1}'.format(tmpStat, tmpMsg))
if tmpStat is not True:
return tmpStat, tmpMsg
# Now that output files have been all copied to Local RSE register transient dataset
# loop over all transfers
tmpStat = None
tmpMsg = ''
srcRSE = None
dstRSE = None
datasetName = 'panda.harvester.{0}.{1}'.format(jobspec.PandaID,str(uuid.uuid4()))
datasetScope = 'transient'
# get destination endpoint
nucleus = jobspec.jobParams['nucleus']
agis = self.dbInterface.get_cache('panda_queues.json').data
dstRSE = [agis[x]["astorages"]['pr'][0] for x in agis if agis[x]["atlas_site"] == nucleus][0]
# get the list of output files to transfer
fileSpecs = jobspec.get_output_file_specs(skip_done=True)
fileList = []
lfns = []
for fileSpec in fileSpecs:
tmpFile = dict()
tmpFile['scope'] = datasetScope
tmpFile['name'] = fileSpec.lfn
tmpFile['bytes'] = fileSpec.fsize
tmpFile['adler32'] = fileSpec.chksum
tmpFile['meta'] = {'guid': fileSpec.fileAttributes['guid']}
fileList.append(tmpFile)
lfns.append(fileSpec.lfn)
# get source RSE
if srcRSE is None and fileSpec.objstoreID is not None:
ddm = self.dbInterface.get_cache('agis_ddmendpoints.json').data
srcRSE = [x for x in ddm if ddm[x]["id"] == fileSpec.objstoreID][0]
# test that srcRSE and dstRSE are defined
errStr = ''
if srcRSE is None:
errStr = 'Source RSE is not defined '
if dstRSE is None:
errStr = errStr + ' Desitination RSE is not defined'
if (srcRSE is None) or (dstRSE is None) :
tmpLog.error(errStr)
return False,errStr
# create the dataset and add files to it and create a transfer rule
try:
# register dataset
tmpLog.debug('register {0}:{1}'.format(datasetScope, datasetName))
rucioAPI = RucioClient()
try:
rucioAPI.add_dataset(datasetScope, datasetName,
meta={'hidden': True},
lifetime=30 * 24 * 60 * 60,
rse=srcRSE
)
except DataIdentifierAlreadyExists:
# ignore even if the dataset already exists
pass
except Exception:
tmpLog.error('Could not create dataset with scope: {0} Name: {1} in Rucio'
.format(datasetScope,datasetName))
raise
# add files to dataset
try:
rucioAPI.add_files_to_datasets([{'scope': datasetScope,
'name': datasetName,
'dids': fileList,
'rse': srcRSE}],
ignore_duplicate=True)
except FileAlreadyExists:
# ignore if files already exist
pass
except Exception:
tmpLog.error('Could add files to dataset with scope: {0} Name: {1} in Rucio'
.format(datasetScope,datasetName))
raise
# add rule
try:
tmpDID = dict()
tmpDID['scope'] = datasetScope
tmpDID['name'] = datasetName
tmpRet = rucioAPI.add_replication_rule([tmpDID], 1, dstRSE,
lifetime=30 * 24 * 60 * 60)
ruleIDs = tmpRet[0]
tmpLog.debug('registered dataset {0}:{1} with rule {2}'.format(datasetScope, datasetName,
str(ruleIDs)))
# group the output files together by the Rucio transfer rule
jobspec.set_groups_to_files({ruleIDs: {'lfns': lfns,'groupStatus': 'pending'}})
msgStr = 'jobspec.set_groups_to_files -Rucio rule - {0}, lfns - {1}, groupStatus - pending'.format(ruleIDs,lfns)
tmpLog.debug(msgStr)
#.........这里部分代码省略.........