本文整理汇总了Python中WMQuality.Emulators.DataBlockGenerator.DataBlockGenerator.DataBlockGenerator.getBlocks方法的典型用法代码示例。如果您正苦于以下问题:Python DataBlockGenerator.getBlocks方法的具体用法?Python DataBlockGenerator.getBlocks怎么用?Python DataBlockGenerator.getBlocks使用的例子?那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类WMQuality.Emulators.DataBlockGenerator.DataBlockGenerator.DataBlockGenerator
的用法示例。
在下文中一共展示了DataBlockGenerator.getBlocks方法的6个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: listFiles
# 需要导入模块: from WMQuality.Emulators.DataBlockGenerator.DataBlockGenerator import DataBlockGenerator [as 别名]
# 或者: from WMQuality.Emulators.DataBlockGenerator.DataBlockGenerator.DataBlockGenerator import getBlocks [as 别名]
def listFiles(self, datasetPath, retriveList):
res = []
dbg = DataBlockGenerator()
for block in dbg.getBlocks(datasetPath):
files = dbg.getFiles(block['Name'])
for f in files:
f['Block'] = block
res.append(f)
return res
示例2: __init__
# 需要导入模块: from WMQuality.Emulators.DataBlockGenerator.DataBlockGenerator import DataBlockGenerator [as 别名]
# 或者: from WMQuality.Emulators.DataBlockGenerator.DataBlockGenerator.DataBlockGenerator import getBlocks [as 别名]
class DBSReader:
"""
Mock up dbs access
"""
def __init__(self, *args, **kwargs):
print "Using DBS Emulator ..."
self.dataBlocks = DataBlockGenerator()
def getFileBlocksInfo(self, dataset, onlyClosedBlocks = True):
"""Fake block info"""
return self.dataBlocks.getBlocks(dataset)
def listFileBlockLocation(self, block):
"""Fake locations"""
return self.dataBlocks.getLocation(block)
def listFilesInBlock(self, block):
"""Fake files"""
return self.dataBlocks.getFiles(block)
def getFileBlock(self, block):
"""Return block + locations"""
result = { block : {
"StorageElements" : self.listFileBlockLocation(block),
"Files" : self.listFilesInBlock(block),
"IsOpen" : False,
}
}
return result
def getDatasetInfo(self, dataset):
"""Dataset summary"""
result = {}
result['number_of_events'] = sum([x['NumberOfEvents']
for x in self.dataBlocks.getBlocks(dataset)])
result['number_of_files'] = sum([x['NumberOfFiles']
for x in self.dataBlocks.getBlocks(dataset)])
result['path'] = dataset
return result
示例3: MockPhEDExApi
# 需要导入模块: from WMQuality.Emulators.DataBlockGenerator.DataBlockGenerator import DataBlockGenerator [as 别名]
# 或者: from WMQuality.Emulators.DataBlockGenerator.DataBlockGenerator.DataBlockGenerator import getBlocks [as 别名]
#.........这里部分代码省略.........
for block in blockList:
if isinstance(block, dict):
block = block['Name']
dataset = self.dataBlocks.getDatasetName(block)
# TODO needs to add correct file numbers
datasetList = data['phedex']['dataset']
if datasetList:
find = False
for dataItem in datasetList:
if dataItem['name'] == dataset:
datasetSelected = dataItem
find = True
break
if not (datasetList and find):
data['phedex']['dataset'].append({'name': dataset, 'files': FILES_PER_DATASET,
'block': []})
datasetSelected = data['phedex']['dataset'][-1]
subs = []
subs.append({'node': 'T2_XX_SiteA_MSS', 'custodial': 'n', 'suspend_until': None,
'level': 'dataset', 'move': 'n', 'request': '47983',
'time_created': '1232989000', 'priority': 'low',
'time_update': None, 'node_id': '781',
'suspended': 'n', 'group': None})
if dataset in self.subRequests:
subs.extend(self.subRequests[dataset])
datasetSelected['subscription'] = subs
for sub in subs:
if sub['level'] == 'block':
subs.remove(sub)
blocks = datasetSelected['block']
locations = self.dataBlocks.getLocation(block)
blocks.append({"bytes": "10438786614", "files": FILES_PER_BLOCK, "is_open": "n", "name": block,
"id": "454370",
"subscription": [{'node': x + '_MSS', "suspended": "n"} for x in locations]
})
data = {'phedex': {"request_timestamp": 1254850198.15418,
'dataset': []}}
# Different structure depending on whether we ask for dataset or blocks
if 'dataset' in args and args['dataset']:
blockList = self.dataBlocks.getBlocks(args['dataset'])
_blockInfoGenerator(blockList)
elif 'block' in args and args['block']:
_blockInfoGenerator(args['block'])
elif 'group' in args and args['group']:
blockList = self.dataBlocks.getBlocks('/a/b-%s/c' % args['group'])
_blockInfoGenerator(blockList)
return data
def getRequestList(self, **kwargs):
"""
_getRequestList_
Emulated request list, for now it does nothing
"""
goldenResponse = {"phedex": {"request": [], "request_timestamp": 1368636296.94707,
"request_version": "2.3.15-comp", "request_call": "requestlist",
"call_time": 0.34183, "request_date": "2013-05-15 16:44:56 UTC"}}
return goldenResponse
def __getattr__(self, item):
"""
__getattr__ gets called in case lookup of the actual method fails. We use this to return data based on
a lookup table
:param item: The method name the user is trying to call
:return: The generic lookup function
"""
def genericLookup(*args, **kwargs):
"""
This function returns the mocked DBS data
:param args: positional arguments it was called with
:param kwargs: named arguments it was called with
:return: the dictionary that DBS would have returned
"""
if kwargs:
signature = '%s:%s' % (item, sorted(kwargs.iteritems()))
else:
signature = item
try:
if MOCK_DATA[self.url][signature] == 'Raises HTTPError':
raise HTTPError
else:
return MOCK_DATA[self.url][signature]
except KeyError:
raise KeyError("PhEDEx mock API could not return data for method %s, args=%s, and kwargs=%s (URL %s)." %
(item, args, kwargs, self.url))
return genericLookup
示例4: PhEDEx
# 需要导入模块: from WMQuality.Emulators.DataBlockGenerator.DataBlockGenerator import DataBlockGenerator [as 别名]
# 或者: from WMQuality.Emulators.DataBlockGenerator.DataBlockGenerator.DataBlockGenerator import getBlocks [as 别名]
#.........这里部分代码省略.........
'block' : []})
datasetSelected = data['phedex']['dataset'][-1]
subs = []
subs.append({'node': 'T2_XX_SiteA_MSS', 'custodial': 'n', 'suspend_until': None,
'level': 'dataset', 'move': 'n', 'request': '47983',
'time_created': '1232989000', 'priority': 'low',
'time_update': None, 'node_id': '781',
'suspended': 'n', 'group': None})
# subs.append({'node': 'T2_XX_SiteB', 'custodial': 'n', 'suspend_until': None,
# 'level': 'dataset', 'move': 'n', 'request': '47983',
# 'time_created': '1232989000', 'priority': 'low',
# 'time_update': None, 'node_id': '781',
# 'suspended': 'n', 'group': None})
datasetSelected['subscription'] = subs
blocks = datasetSelected['block']
locations= self.dataBlocks.getLocation(block)
blocks.append({"bytes":"10438786614",
"files":filesInBlock,
"is_open":"n",
"name": block,
"id":"454370", "subscription"
:[ {'node' : x + '_MSS', "suspended" : "n"} for x in locations]
#{"priority":"normal", "request":"51253", "time_created":"1245165314",
# "move":"n", "suspend_until":None, "node":"T2_XX_SiteA",
# "time_update":"1228905272", "group":None, "level":"block",
# "node_id":"641", "custodial":"n", "suspended":"n"}]
})
data = {'phedex' : {"request_timestamp" : 1254850198.15418,
'dataset' : []}}
# different structure depending on whether we ask for dataset or blocks
if args.has_key('dataset') and args['dataset']:
for dataset in args['dataset']:
blockList = self.dataBlocks.getBlocks(dataset)
_blockInfoGenerator(blockList)
elif args.has_key('block') and args['block']:
_blockInfoGenerator(args['block'])
return data
def getSubscriptionMapping(self, *dataItems, **kwargs):
"""
Similar basic functionality as self.subscriptions()
however: dataItems may be a combination of blocks or datasets and
kwargs is passed to PhEDEx; output is parsed and returned in the form
{ 'dataItem1' : [Node1, Node2] } where dataItem is a block or dataset
The following cases are handled:
o Input is a block and subscription is a dataset
o Input is a block and subscription is a block
o Input is a dataset and subscription is a dataset
Not supported:
o Input is a dataset but only block subscriptions exist
"""
from collections import defaultdict
result = defaultdict(set)
kwargs.setdefault('suspended', 'n') # require active subscription
dataItems = list(set(dataItems)) # force unique items
# Hard to query all at once in one GET call, POST not cacheable
# hence, query individually - use httplib2 caching to protect service
for item in dataItems:
# First query for a dataset level subscription (most common)
# this returns block level subscriptions also.
# Rely on httplib2 caching to not resend on every block in dataset
kwargs['dataset'], kwargs['block'] = [item.split('#')[0]], []
response = self.subscriptions(**kwargs)['phedex']
# iterate over response as can't jump to specific datasets
for dset in response['dataset']:
if dset['name'] != item.split('#')[0]:
continue
if dset.has_key('subscription'):
# dataset level subscription
nodes = [x['node'] for x in dset['subscription']
if x['suspended'] == 'n']
result[item].update(nodes)
#if we have a block we must check for block level subscription also
# combine with original query when can give both dataset and block
if item.find('#') > -1 and dset.has_key('block'):
for block in dset['block']:
if block['name'] == item:
nodes = [x['node'] for x in block['subscription']
if x['suspended'] == 'n']
result[item].update(nodes)
break
return result
def emulator(self):
return "PhEDEx emulator ...."
示例5: __init__
# 需要导入模块: from WMQuality.Emulators.DataBlockGenerator.DataBlockGenerator import DataBlockGenerator [as 别名]
# 或者: from WMQuality.Emulators.DataBlockGenerator.DataBlockGenerator.DataBlockGenerator import getBlocks [as 别名]
class DBSReader:
"""
Mock up dbs access
"""
def __init__(self, url, **contact):
self.dataBlocks = DataBlockGenerator()
args = { "url" : url, "level" : 'ERROR', "version" : 'DBS_2_0_9'}
self.dbs = _MockDBSApi(args)
def getFileBlocksInfo(self, dataset, onlyClosedBlocks = True,
blockName = '*', locations = True):
"""Fake block info"""
blocks = [x for x in self.dataBlocks.getBlocks(dataset)
if x['Name'] == blockName or blockName == '*']
if not blocks:
# Weird error handling follows, this is what dbs does:
# If block specified, return [], else raise DbsBadRequest error
if blockName != '*':
return []
else:
raise DBSReaderError('DbsBadRequest: DBS Server Raised An Error')
if locations:
for block in blocks:
block['StorageElementList'] = [{'Role' : '', 'Name' : x} for x in \
self.listFileBlockLocation(block['Name'])]
return blocks
def lfnsInBlock(self, fileBlockName):
"""
_lfnsInBlock_
Get a fake list of LFNs for the block
"""
files = self.listFilesInBlock(fileBlockName)
return [x['LogicalFileName'] for x in files]
def listFileBlocks(self, dataset, onlyClosedBlocks = False,
blockName = '*'):
"""Get fake block names"""
return [x['Name'] for x in self.getFileBlocksInfo(dataset, onlyClosedBlocks = False,
blockName = blockName,
locations = False)]
def listOpenFileBlocks(self, dataset):
"""
_listOpenFileBlocks_
Retrieve a list of open fileblock names for a dataset
"""
return [x['Name'] for x in self.getFileBlocksInfo(dataset, onlyClosedBlocks = False,
locations = False) if str(x['OpenForWriting' ]) == '1']
def listFileBlockLocation(self, block):
"""Fake locations"""
return self.dataBlocks.getLocation(block)
def listFilesInBlock(self, fileBlockName):
"""Fake files"""
return self.dataBlocks.getFiles(fileBlockName)
def listFilesInBlockWithParents(self, block):
return self.dataBlocks.getFiles(block, True)
def getFileBlock(self, block):
"""Return block + locations"""
result = { block : {
"StorageElements" : self.listFileBlockLocation(block),
"Files" : self.listFilesInBlock(block),
"IsOpen" : self.dataBlocks._openForWriting(),
}
}
return result
def getFileBlockWithParents(self, fileBlockName):
"""
_getFileBlockWithParents_
return a dictionary:
{ blockName: {
"StorageElements" : [<se list>],
"Files" : dictionaries representing each file
}
}
files
"""
result = { fileBlockName: {
"StorageElements" : self.listFileBlockLocation(fileBlockName),
"Files" : self.listFilesInBlockWithParents(fileBlockName),
"IsOpen" : self.dataBlocks._openForWriting(),
}
}
return result
#.........这里部分代码省略.........
示例6: __init__
# 需要导入模块: from WMQuality.Emulators.DataBlockGenerator.DataBlockGenerator import DataBlockGenerator [as 别名]
# 或者: from WMQuality.Emulators.DataBlockGenerator.DataBlockGenerator.DataBlockGenerator import getBlocks [as 别名]
class DBSReader:
"""
Mock up dbs access
"""
def __init__(self, url, **contact):
self.dataBlocks = DataBlockGenerator()
args = { "url" : url, "level" : 'ERROR', "version" : 'DBS_2_0_9'}
self.dbs = _MockDBSApi(args)
def getFileBlocksInfo(self, dataset, onlyClosedBlocks = True,
blockName = '*', locations = True):
"""Fake block info"""
blocks = [x for x in self.dataBlocks.getBlocks(dataset)
if x['Name'] == blockName or blockName == '*']
if locations:
for block in blocks:
block['StorageElementList'] = [{'Role' : '', 'Name' : x} for x in \
self.listFileBlockLocation(block['Name'])]
return blocks
def listFileBlockLocation(self, block):
"""Fake locations"""
return self.dataBlocks.getLocation(block)
def listFilesInBlock(self, block):
"""Fake files"""
return self.dataBlocks.getFiles(block)
def listFilesInBlockWithParents(self, block):
return self.dataBlocks.getFiles(block, True)
def getFileBlock(self, block):
"""Return block + locations"""
result = { block : {
"StorageElements" : self.listFileBlockLocation(block),
"Files" : self.listFilesInBlock(block),
"IsOpen" : False,
}
}
return result
def getFileBlockWithParents(self, fileBlockName):
"""
_getFileBlockWithParents_
return a dictionary:
{ blockName: {
"StorageElements" : [<se list>],
"Files" : dictionaries representing each file
}
}
files
"""
result = { fileBlockName: {
"StorageElements" : self.listFileBlockLocation(fileBlockName),
"Files" : self.listFilesInBlockWithParents(fileBlockName),
"IsOpen" : False,
}
}
return result
def listRuns(self, dataset = None, block = None):
def getRunsFromBlock(b):
results = []
for x in self.dataBlocks.getFiles(b):
results.extend([y['RunNumber'] for y in x['LumiList']])
return results
if block:
return getRunsFromBlock(block)
if dataset:
runs = []
for block in self.dataBlocks.getBlocks(dataset):
runs.extend(getRunsFromBlock(block['Name']))
return runs
return None
def getDBSSummaryInfo(self, dataset=None, block=None):
"""Dataset summary"""
def getLumisectionsInBlock(b):
lumis = set()
for file in self.dataBlocks.getFiles(b):
for x in file['LumiList']:
lumis.add(x['LumiSectionNumber'])
return lumis
result = {}
if block:
result['NumberOfEvents'] = sum([x['NumberOfEvents']
for x in self.dataBlocks.getFiles(block)])
result['NumberOfFiles'] = len(self.dataBlocks.getFiles(block))
result['NumberOfLumis'] = len(getLumisectionsInBlock(block))
#.........这里部分代码省略.........