本文整理汇总了Python中WMQuality.Emulators.DataBlockGenerator.DataBlockGenerator.DataBlockGenerator.getFiles方法的典型用法代码示例。如果您正苦于以下问题：Python DataBlockGenerator.getFiles方法的具体用法？Python DataBlockGenerator.getFiles怎么用？Python DataBlockGenerator.getFiles使用的例子？那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类WMQuality.Emulators.DataBlockGenerator.DataBlockGenerator.DataBlockGenerator的用法示例。
在下文中一共展示了DataBlockGenerator.getFiles方法的6个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: listFiles
# 需要导入模块: from WMQuality.Emulators.DataBlockGenerator.DataBlockGenerator import DataBlockGenerator [as 别名]
# 或者: from WMQuality.Emulators.DataBlockGenerator.DataBlockGenerator.DataBlockGenerator import getFiles [as 别名]
def listFiles(self, datasetPath, retriveList):
res = []
dbg = DataBlockGenerator()
for block in dbg.getBlocks(datasetPath):
files = dbg.getFiles(block['Name'])
for f in files:
f['Block'] = block
res.append(f)
return res示例2: __init__
# 需要导入模块: from WMQuality.Emulators.DataBlockGenerator.DataBlockGenerator import DataBlockGenerator [as 别名]
# 或者: from WMQuality.Emulators.DataBlockGenerator.DataBlockGenerator.DataBlockGenerator import getFiles [as 别名]
class DBSReader:
"""
Mock up dbs access
"""
def __init__(self, *args, **kwargs):
print "Using DBS Emulator ..."
self.dataBlocks = DataBlockGenerator()
def getFileBlocksInfo(self, dataset, onlyClosedBlocks = True):
"""Fake block info"""
return self.dataBlocks.getBlocks(dataset)
def listFileBlockLocation(self, block):
"""Fake locations"""
return self.dataBlocks.getLocation(block)
def listFilesInBlock(self, block):
"""Fake files"""
return self.dataBlocks.getFiles(block)
def getFileBlock(self, block):
"""Return block + locations"""
result = { block : {
"StorageElements" : self.listFileBlockLocation(block),
"Files" : self.listFilesInBlock(block),
"IsOpen" : False,
}
}
return result
def getDatasetInfo(self, dataset):
"""Dataset summary"""
result = {}
result['number_of_events'] = sum([x['NumberOfEvents']
for x in self.dataBlocks.getBlocks(dataset)])
result['number_of_files'] = sum([x['NumberOfFiles']
for x in self.dataBlocks.getBlocks(dataset)])
result['path'] = dataset
return result示例3: PhEDEx
# 需要导入模块: from WMQuality.Emulators.DataBlockGenerator.DataBlockGenerator import DataBlockGenerator [as 别名]
# 或者: from WMQuality.Emulators.DataBlockGenerator.DataBlockGenerator.DataBlockGenerator import getFiles [as 别名]
#.........这里部分代码省略.........
"kind" : "MSS",
"se" : "srm-cms.gridpp.rl.ac.uk",
"technology" : "Castor",
"id" : 3})
nodeMappings["phedex"]["node"].append({"name" : "T1_UK_RAL_Buffer",
"kind" : "Buffer",
"se" : "srm-cms.gridpp.rl.ac.uk",
"technology" : "Castor",
"id" : 4})
nodeMappings["phedex"]["node"].append({"name" : "T1_UK_RAL_Disk",
"kind" : "Disk",
"se" : "srm-cms-disk.gridpp.rl.ac.uk",
"technology" : "Disk",
"id" : 5})
nodeMappings["phedex"]["node"].append({"name" : "T2_CH_CERN",
"kind" : "Disk",
"se" : "srm-eoscms.cern.ch",
"technology" : "Disk",
"id" : 6})
nodeMappings["phedex"]["node"].append({"name" : "T3_CO_Uniandes",
"kind" : "Disk",
"se" : "moboro.uniandes.edu.co",
"technology" : "DPM",
"id" : 7})
return nodeMappings
def getReplicaInfoForBlocks(self, **args):
"""
Where are blocks located
"""
data = {"phedex":{"request_timestamp":1254762796.13538, "block" : []}}
for block in args['block']:
blocks = data['phedex']['block']
files = self.dataBlocks.getFiles(block)
locations = self.dataBlocks.getLocation(block)
blocks.append({"files": len(files), "name": block,
'replica' : [{'node' : x + '_MSS' } for x in locations]})
return data
def subscriptions(self, **args):
"""
Where is data subscribed - for now just replicate blockreplicas
"""
def _blockInfoGenerator(blockList):
for block in blockList:
if type(block) == dict:
block = block['Name']
dataset = self.dataBlocks.getDatasetName(block)
# TODO needs to add correct file numbers
datasetList = data['phedex']['dataset']
if datasetList:
find = False
for dataItem in datasetList:
if dataItem['name'] == dataset:
datasetSelected = dataItem
find = True
break
if not datasetList or find:
data['phedex']['dataset'].append({'name' : dataset, 'files' : filesInDataset,
'block' : []})
datasetSelected = data['phedex']['dataset'][-1]
subs = []示例4: __init__
# 需要导入模块: from WMQuality.Emulators.DataBlockGenerator.DataBlockGenerator import DataBlockGenerator [as 别名]
# 或者: from WMQuality.Emulators.DataBlockGenerator.DataBlockGenerator.DataBlockGenerator import getFiles [as 别名]
class DBSReader:
"""
Mock up dbs access
"""
def __init__(self, url, **contact):
self.dataBlocks = DataBlockGenerator()
args = { "url" : url, "level" : 'ERROR', "version" : 'DBS_2_0_9'}
self.dbs = _MockDBSApi(args)
def getFileBlocksInfo(self, dataset, onlyClosedBlocks = True,
blockName = '*', locations = True):
"""Fake block info"""
blocks = [x for x in self.dataBlocks.getBlocks(dataset)
if x['Name'] == blockName or blockName == '*']
if not blocks:
# Weird error handling follows, this is what dbs does:
# If block specified, return [], else raise DbsBadRequest error
if blockName != '*':
return []
else:
raise DBSReaderError('DbsBadRequest: DBS Server Raised An Error')
if locations:
for block in blocks:
block['StorageElementList'] = [{'Role' : '', 'Name' : x} for x in \
self.listFileBlockLocation(block['Name'])]
return blocks
def lfnsInBlock(self, fileBlockName):
"""
_lfnsInBlock_
Get a fake list of LFNs for the block
"""
files = self.listFilesInBlock(fileBlockName)
return [x['LogicalFileName'] for x in files]
def listFileBlocks(self, dataset, onlyClosedBlocks = False,
blockName = '*'):
"""Get fake block names"""
return [x['Name'] for x in self.getFileBlocksInfo(dataset, onlyClosedBlocks = False,
blockName = blockName,
locations = False)]
def listOpenFileBlocks(self, dataset):
"""
_listOpenFileBlocks_
Retrieve a list of open fileblock names for a dataset
"""
return [x['Name'] for x in self.getFileBlocksInfo(dataset, onlyClosedBlocks = False,
locations = False) if str(x['OpenForWriting' ]) == '1']
def listFileBlockLocation(self, block):
"""Fake locations"""
return self.dataBlocks.getLocation(block)
def listFilesInBlock(self, fileBlockName):
"""Fake files"""
return self.dataBlocks.getFiles(fileBlockName)
def listFilesInBlockWithParents(self, block):
return self.dataBlocks.getFiles(block, True)
def getFileBlock(self, block):
"""Return block + locations"""
result = { block : {
"StorageElements" : self.listFileBlockLocation(block),
"Files" : self.listFilesInBlock(block),
"IsOpen" : self.dataBlocks._openForWriting(),
}
}
return result
def getFileBlockWithParents(self, fileBlockName):
"""
_getFileBlockWithParents_
return a dictionary:
{ blockName: {
"StorageElements" : [<se list>],
"Files" : dictionaries representing each file
}
}
files
"""
result = { fileBlockName: {
"StorageElements" : self.listFileBlockLocation(fileBlockName),
"Files" : self.listFilesInBlockWithParents(fileBlockName),
"IsOpen" : self.dataBlocks._openForWriting(),
}
}
return result
#.........这里部分代码省略.........
示例5: __init__
# 需要导入模块: from WMQuality.Emulators.DataBlockGenerator.DataBlockGenerator import DataBlockGenerator [as 别名]
# 或者: from WMQuality.Emulators.DataBlockGenerator.DataBlockGenerator.DataBlockGenerator import getFiles [as 别名]
class PhEDEx:
"""
"""
def __init__(self, *args, **kwargs):
print "Using PhEDEx Emulator ...."
self.dataBlocks = DataBlockGenerator()
def getReplicaInfoForBlocks(self, **args):
"""
Where are blocks located
"""
for block in args['block']:
data = {"phedex":{"request_timestamp":1254762796.13538, "block" : []}}
blocks = data['phedex']['block']
files = self.dataBlocks.getFiles(block)
locations = self.dataBlocks.getLocation(block)
blocks.append({"files": len(files), "name": block,
'replica' : [{'se' : x } for x in locations]})
return data
def subscriptions(self, **args):
"""
Where is data subscribed - for now just replicate blockreplicas
"""
data = {'phedex' : {"request_timestamp" : 1254850198.15418,
'dataset' : []}}
# different structure depending on whether we ask for dataset or blocks
if args.has_key('dataset') and args['dataset']:
for dataset in args['dataset']:
# TODO needs to add correct file numbers
data['phedex']['dataset'].append({'name' : dataset, 'files' : 5,
'subscription' : []})
subs = data['phedex']['dataset'][-1]['subscription']
#FIXME: Take from self.locations
subs.append({'node': 'SiteA', 'custodial': 'n', 'suspend_until': None,
'level': 'dataset', 'move': 'n', 'request': '47983',
'time_created': '1232989000', 'priority': 'low',
'time_update': None, 'node_id': '781',
'suspended': 'n', 'group': None})
return data
elif args.has_key('block') and args['block']:
for block in args['block']:
dataset = self.dataBlocks.getDataset('block')
# TODO needs to add correct file numbers
data['phedex']['dataset'].append({'name' : dataset, 'files' : 5,
'block' : []})
blocks = data['phedex']['dataset'][-1]['block']
locations= self.dataBlocks.getLocation(block)
blocks.append({"bytes":"10438786614", "files":"5", "is_open":"n",
"name": args['block'],
"id":"454370", "subscription"
:[ {'node' : x } for x in locations]
#{"priority":"normal", "request":"51253", "time_created":"1245165314",
# "move":"n", "suspend_until":None, "node":"SiteA",
# "time_update":"1228905272", "group":None, "level":"block",
# "node_id":"641", "custodial":"n", "suspended":"n"}]
})
return data
def emulator(self):
return "PhEDEx emulator ...."示例6: __init__
# 需要导入模块: from WMQuality.Emulators.DataBlockGenerator.DataBlockGenerator import DataBlockGenerator [as 别名]
# 或者: from WMQuality.Emulators.DataBlockGenerator.DataBlockGenerator.DataBlockGenerator import getFiles [as 别名]
class DBSReader:
"""
Mock up dbs access
"""
def __init__(self, url, **contact):
self.dataBlocks = DataBlockGenerator()
args = { "url" : url, "level" : 'ERROR', "version" : 'DBS_2_0_9'}
self.dbs = _MockDBSApi(args)
def getFileBlocksInfo(self, dataset, onlyClosedBlocks = True,
blockName = '*', locations = True):
"""Fake block info"""
blocks = [x for x in self.dataBlocks.getBlocks(dataset)
if x['Name'] == blockName or blockName == '*']
if locations:
for block in blocks:
block['StorageElementList'] = [{'Role' : '', 'Name' : x} for x in \
self.listFileBlockLocation(block['Name'])]
return blocks
def listFileBlockLocation(self, block):
"""Fake locations"""
return self.dataBlocks.getLocation(block)
def listFilesInBlock(self, block):
"""Fake files"""
return self.dataBlocks.getFiles(block)
def listFilesInBlockWithParents(self, block):
return self.dataBlocks.getFiles(block, True)
def getFileBlock(self, block):
"""Return block + locations"""
result = { block : {
"StorageElements" : self.listFileBlockLocation(block),
"Files" : self.listFilesInBlock(block),
"IsOpen" : False,
}
}
return result
def getFileBlockWithParents(self, fileBlockName):
"""
_getFileBlockWithParents_
return a dictionary:
{ blockName: {
"StorageElements" : [<se list>],
"Files" : dictionaries representing each file
}
}
files
"""
result = { fileBlockName: {
"StorageElements" : self.listFileBlockLocation(fileBlockName),
"Files" : self.listFilesInBlockWithParents(fileBlockName),
"IsOpen" : False,
}
}
return result
def listRuns(self, dataset = None, block = None):
def getRunsFromBlock(b):
results = []
for x in self.dataBlocks.getFiles(b):
results.extend([y['RunNumber'] for y in x['LumiList']])
return results
if block:
return getRunsFromBlock(block)
if dataset:
runs = []
for block in self.dataBlocks.getBlocks(dataset):
runs.extend(getRunsFromBlock(block['Name']))
return runs
return None
def getDBSSummaryInfo(self, dataset=None, block=None):
"""Dataset summary"""
def getLumisectionsInBlock(b):
lumis = set()
for file in self.dataBlocks.getFiles(b):
for x in file['LumiList']:
lumis.add(x['LumiSectionNumber'])
return lumis
result = {}
if block:
result['NumberOfEvents'] = sum([x['NumberOfEvents']
for x in self.dataBlocks.getFiles(block)])
result['NumberOfFiles'] = len(self.dataBlocks.getFiles(block))
result['NumberOfLumis'] = len(getLumisectionsInBlock(block))
#.........这里部分代码省略.........