Python DataBlockGenerator.getBlocks方法代码示例

# 需要导入模块: from WMQuality.Emulators.DataBlockGenerator.DataBlockGenerator import DataBlockGenerator [as 别名]
# 或者: from WMQuality.Emulators.DataBlockGenerator.DataBlockGenerator.DataBlockGenerator import getBlocks [as 别名]
    def listFiles(self, datasetPath, retriveList):
        res = []
        dbg = DataBlockGenerator()
        for block in dbg.getBlocks(datasetPath):
            files = dbg.getFiles(block['Name'])
            for f in files:
                f['Block'] = block
                res.append(f)

        return res

# 需要导入模块: from WMQuality.Emulators.DataBlockGenerator.DataBlockGenerator import DataBlockGenerator [as 别名]
# 或者: from WMQuality.Emulators.DataBlockGenerator.DataBlockGenerator.DataBlockGenerator import getBlocks [as 别名]
class DBSReader:
    """
    Mock up dbs access
    """
    def __init__(self, *args, **kwargs):
        print "Using DBS Emulator ..."
        self.dataBlocks = DataBlockGenerator()
        
    def getFileBlocksInfo(self, dataset, onlyClosedBlocks = True):
        """Fake block info"""
        return self.dataBlocks.getBlocks(dataset)

    def listFileBlockLocation(self, block):
        """Fake locations"""
        return self.dataBlocks.getLocation(block)

    def listFilesInBlock(self, block):
        """Fake files"""
        return self.dataBlocks.getFiles(block)

    def getFileBlock(self, block):
        """Return block + locations"""
        result = { block : {
            "StorageElements" : self.listFileBlockLocation(block),
            "Files" : self.listFilesInBlock(block),
            "IsOpen" : False,
            }
                   }
        return result

    def getDatasetInfo(self, dataset):
        """Dataset summary"""
        result = {}
        result['number_of_events'] = sum([x['NumberOfEvents'] 
                                for x in self.dataBlocks.getBlocks(dataset)])
        result['number_of_files'] = sum([x['NumberOfFiles'] 
                                for x in self.dataBlocks.getBlocks(dataset)])
        result['path'] = dataset
        return result

# 需要导入模块: from WMQuality.Emulators.DataBlockGenerator.DataBlockGenerator import DataBlockGenerator [as 别名]
# 或者: from WMQuality.Emulators.DataBlockGenerator.DataBlockGenerator.DataBlockGenerator import getBlocks [as 别名]

#.........这里部分代码省略.........

            for block in blockList:
                if isinstance(block, dict):
                    block = block['Name']
                dataset = self.dataBlocks.getDatasetName(block)
                # TODO needs to add correct file numbers
                datasetList = data['phedex']['dataset']
                if datasetList:
                    find = False
                    for dataItem in datasetList:
                        if dataItem['name'] == dataset:
                            datasetSelected = dataItem
                            find = True
                            break
                if not (datasetList and find):
                    data['phedex']['dataset'].append({'name': dataset, 'files': FILES_PER_DATASET,
                                                      'block': []})

                    datasetSelected = data['phedex']['dataset'][-1]
                subs = []
                subs.append({'node': 'T2_XX_SiteA_MSS', 'custodial': 'n', 'suspend_until': None,
                             'level': 'dataset', 'move': 'n', 'request': '47983',
                             'time_created': '1232989000', 'priority': 'low',
                             'time_update': None, 'node_id': '781',
                             'suspended': 'n', 'group': None})

                if dataset in self.subRequests:
                    subs.extend(self.subRequests[dataset])
                datasetSelected['subscription'] = subs
                for sub in subs:
                    if sub['level'] == 'block':
                        subs.remove(sub)

                blocks = datasetSelected['block']
                locations = self.dataBlocks.getLocation(block)

                blocks.append({"bytes": "10438786614", "files": FILES_PER_BLOCK, "is_open": "n", "name": block,
                               "id": "454370",
                               "subscription": [{'node': x + '_MSS', "suspended": "n"} for x in locations]
                              })

        data = {'phedex': {"request_timestamp": 1254850198.15418,
                           'dataset': []}}

        # Different structure depending on whether we ask for dataset or blocks
        if 'dataset' in args and args['dataset']:
            blockList = self.dataBlocks.getBlocks(args['dataset'])
            _blockInfoGenerator(blockList)
        elif 'block' in args and args['block']:
            _blockInfoGenerator(args['block'])
        elif 'group' in args and args['group']:
            blockList = self.dataBlocks.getBlocks('/a/b-%s/c' % args['group'])
            _blockInfoGenerator(blockList)

        return data

    def getRequestList(self, **kwargs):
        """
        _getRequestList_
        Emulated request list, for now it does nothing
        """

        goldenResponse = {"phedex": {"request": [], "request_timestamp": 1368636296.94707,
                                     "request_version": "2.3.15-comp", "request_call": "requestlist",
                                     "call_time": 0.34183, "request_date": "2013-05-15 16:44:56 UTC"}}
        return goldenResponse

    def __getattr__(self, item):
        """
        __getattr__ gets called in case lookup of the actual method fails. We use this to return data based on
        a lookup table

        :param item: The method name the user is trying to call
        :return: The generic lookup function
        """

        def genericLookup(*args, **kwargs):
            """
            This function returns the mocked DBS data

            :param args: positional arguments it was called with
            :param kwargs: named arguments it was called with
            :return: the dictionary that DBS would have returned
            """

            if kwargs:
                signature = '%s:%s' % (item, sorted(kwargs.iteritems()))
            else:
                signature = item

            try:
                if MOCK_DATA[self.url][signature] == 'Raises HTTPError':
                    raise HTTPError
                else:
                    return MOCK_DATA[self.url][signature]
            except KeyError:
                raise KeyError("PhEDEx mock API could not return data for method %s, args=%s, and kwargs=%s (URL %s)." %
                               (item, args, kwargs, self.url))

        return genericLookup

# 需要导入模块: from WMQuality.Emulators.DataBlockGenerator.DataBlockGenerator import DataBlockGenerator [as 别名]
# 或者: from WMQuality.Emulators.DataBlockGenerator.DataBlockGenerator.DataBlockGenerator import getBlocks [as 别名]

#.........这里部分代码省略.........
                                                      'block' : []})


                    datasetSelected = data['phedex']['dataset'][-1]
                subs = []
                subs.append({'node': 'T2_XX_SiteA_MSS', 'custodial': 'n', 'suspend_until': None,
                                 'level': 'dataset', 'move': 'n', 'request': '47983',
                                 'time_created': '1232989000', 'priority': 'low',
                                 'time_update': None, 'node_id': '781',
                                 'suspended': 'n', 'group': None})
#                subs.append({'node': 'T2_XX_SiteB', 'custodial': 'n', 'suspend_until': None,
#                                 'level': 'dataset', 'move': 'n', 'request': '47983',
#                                 'time_created': '1232989000', 'priority': 'low',
#                                 'time_update': None, 'node_id': '781',
#                                 'suspended': 'n', 'group': None})
                datasetSelected['subscription'] = subs

                blocks = datasetSelected['block']
                locations= self.dataBlocks.getLocation(block)

                blocks.append({"bytes":"10438786614",
                               "files":filesInBlock,
                               "is_open":"n",
                               "name": block,
                               "id":"454370", "subscription"
                                                  :[ {'node' : x + '_MSS', "suspended" : "n"} for x in locations]
                                                        #{"priority":"normal", "request":"51253", "time_created":"1245165314",
                                                        #   "move":"n", "suspend_until":None, "node":"T2_XX_SiteA",
                                                        #   "time_update":"1228905272", "group":None, "level":"block",
                                                        #   "node_id":"641", "custodial":"n", "suspended":"n"}]
                                                    })

        data = {'phedex' : {"request_timestamp" : 1254850198.15418,
                            'dataset' : []}}
        # different structure depending on whether we ask for dataset or blocks

        if args.has_key('dataset') and args['dataset']:
            for dataset in args['dataset']:
                blockList = self.dataBlocks.getBlocks(dataset)
                _blockInfoGenerator(blockList)
        elif args.has_key('block') and args['block']:
            _blockInfoGenerator(args['block'])

        return data

    def getSubscriptionMapping(self, *dataItems, **kwargs):
        """
        Similar basic functionality as self.subscriptions()
        however: dataItems may be a combination of blocks or datasets and
        kwargs is passed to PhEDEx; output is parsed and returned in the form
        { 'dataItem1' : [Node1, Node2] } where dataItem is a block or dataset

        The following cases are handled:
          o Input is a block and subscription is a dataset
          o Input is a block and subscription is a block
          o Input is a dataset and subscription is a dataset

        Not supported:
          o Input is a dataset but only block subscriptions exist
        """
        from collections import defaultdict
        result = defaultdict(set)
        kwargs.setdefault('suspended', 'n') # require active subscription

        dataItems = list(set(dataItems)) # force unique items

        # Hard to query all at once in one GET call, POST not cacheable
        # hence, query individually - use httplib2 caching to protect service
        for item in dataItems:

            # First query for a dataset level subscription (most common)
            # this returns block level subscriptions also.
            # Rely on httplib2 caching to not resend on every block in dataset
            kwargs['dataset'], kwargs['block'] = [item.split('#')[0]], []
            response = self.subscriptions(**kwargs)['phedex']

            # iterate over response as can't jump to specific datasets
            for dset in response['dataset']:
                if dset['name'] != item.split('#')[0]:
                    continue
                if dset.has_key('subscription'):
                    # dataset level subscription
                    nodes = [x['node'] for x in dset['subscription']
                             if x['suspended'] == 'n']
                    result[item].update(nodes)

                #if we have a block we must check for block level subscription also
                # combine with original query when can give both dataset and block
                if item.find('#') > -1 and dset.has_key('block'):
                    for block in dset['block']:
                        if block['name'] == item:
                            nodes = [x['node'] for x in block['subscription']
                                     if x['suspended'] == 'n']
                            result[item].update(nodes)
                            break
        return result


    def emulator(self):
        return "PhEDEx emulator ...."

# 需要导入模块: from WMQuality.Emulators.DataBlockGenerator.DataBlockGenerator import DataBlockGenerator [as 别名]
# 或者: from WMQuality.Emulators.DataBlockGenerator.DataBlockGenerator.DataBlockGenerator import getBlocks [as 别名]
class DBSReader:
    """
    Mock up dbs access
    """
    def __init__(self, url, **contact):
        self.dataBlocks = DataBlockGenerator()
        args = { "url" : url, "level" : 'ERROR', "version" : 'DBS_2_0_9'}
        self.dbs = _MockDBSApi(args)

    def getFileBlocksInfo(self, dataset, onlyClosedBlocks = True,
                          blockName = '*', locations = True):

        """Fake block info"""
        blocks = [x for x in self.dataBlocks.getBlocks(dataset)
                if x['Name'] == blockName or blockName == '*']
        if not blocks:
            # Weird error handling follows, this is what dbs does:
            # If block specified, return [], else raise DbsBadRequest error
            if blockName != '*':
                return []
            else:
                raise DBSReaderError('DbsBadRequest: DBS Server Raised An Error')
        if locations:
            for block in blocks:
                block['StorageElementList'] = [{'Role' : '', 'Name' : x} for x in \
                                               self.listFileBlockLocation(block['Name'])]
        return blocks

    def lfnsInBlock(self, fileBlockName):
        """
        _lfnsInBlock_
        Get a fake list of LFNs for the block
        """

        files = self.listFilesInBlock(fileBlockName)

        return [x['LogicalFileName'] for x in files]

    def listFileBlocks(self, dataset, onlyClosedBlocks = False,
                       blockName = '*'):
        """Get fake block names"""
        return [x['Name'] for x in self.getFileBlocksInfo(dataset, onlyClosedBlocks = False,
                                                          blockName = blockName,
                                                          locations = False)]

    def listOpenFileBlocks(self, dataset):
        """
        _listOpenFileBlocks_

        Retrieve a list of open fileblock names for a dataset

        """
        return [x['Name'] for x in self.getFileBlocksInfo(dataset, onlyClosedBlocks = False,
                                                          locations = False) if str(x['OpenForWriting' ]) == '1']

    def listFileBlockLocation(self, block):
        """Fake locations"""
        return self.dataBlocks.getLocation(block)

    def listFilesInBlock(self, fileBlockName):
        """Fake files"""
        return self.dataBlocks.getFiles(fileBlockName)

    def listFilesInBlockWithParents(self, block):
        return self.dataBlocks.getFiles(block, True)

    def getFileBlock(self, block):
        """Return block + locations"""
        result = { block : {
            "StorageElements" : self.listFileBlockLocation(block),
            "Files" : self.listFilesInBlock(block),
            "IsOpen" : self.dataBlocks._openForWriting(),
            }
                }
        return result

    def getFileBlockWithParents(self, fileBlockName):
        """
        _getFileBlockWithParents_

        return a dictionary:
        { blockName: {
             "StorageElements" : [<se list>],
             "Files" : dictionaries representing each file
             }
        }

        files

        """

        result = { fileBlockName: {
            "StorageElements" : self.listFileBlockLocation(fileBlockName),
            "Files" : self.listFilesInBlockWithParents(fileBlockName),
            "IsOpen" : self.dataBlocks._openForWriting(),

            }
                   }
        return result

#.........这里部分代码省略.........

# 需要导入模块: from WMQuality.Emulators.DataBlockGenerator.DataBlockGenerator import DataBlockGenerator [as 别名]
# 或者: from WMQuality.Emulators.DataBlockGenerator.DataBlockGenerator.DataBlockGenerator import getBlocks [as 别名]
class DBSReader:
    """
    Mock up dbs access
    """
    def __init__(self, url, **contact):
        self.dataBlocks = DataBlockGenerator()
        args = { "url" : url, "level" : 'ERROR', "version" : 'DBS_2_0_9'}
        self.dbs = _MockDBSApi(args)
        
    def getFileBlocksInfo(self, dataset, onlyClosedBlocks = True,
                          blockName = '*', locations = True):

        """Fake block info"""
        blocks = [x for x in self.dataBlocks.getBlocks(dataset)
                if x['Name'] == blockName or blockName == '*']
        if locations:
            for block in blocks:
                block['StorageElementList'] = [{'Role' : '', 'Name' : x} for x in \
                                               self.listFileBlockLocation(block['Name'])]
        return blocks

    def listFileBlockLocation(self, block):
        """Fake locations"""
        return self.dataBlocks.getLocation(block)

    def listFilesInBlock(self, block):
        """Fake files"""
        return self.dataBlocks.getFiles(block)

    def listFilesInBlockWithParents(self, block):
        return self.dataBlocks.getFiles(block, True)

    def getFileBlock(self, block):
        """Return block + locations"""
        result = { block : {
            "StorageElements" : self.listFileBlockLocation(block),
            "Files" : self.listFilesInBlock(block),
            "IsOpen" : False,
            }
                }
        return result

    def getFileBlockWithParents(self, fileBlockName):
        """
        _getFileBlockWithParents_

        return a dictionary:
        { blockName: {
             "StorageElements" : [<se list>],
             "Files" : dictionaries representing each file
             }
        }

        files

        """

        result = { fileBlockName: {
            "StorageElements" : self.listFileBlockLocation(fileBlockName),
            "Files" : self.listFilesInBlockWithParents(fileBlockName),
            "IsOpen" : False,

            }
                   }
        return result

    def listRuns(self, dataset = None, block = None):
        def getRunsFromBlock(b):
            results = []
            for x in self.dataBlocks.getFiles(b):
                results.extend([y['RunNumber'] for y in x['LumiList']])
            return results

        if block:
            return getRunsFromBlock(block)
        if dataset:
            runs = []
            for block in self.dataBlocks.getBlocks(dataset):
                runs.extend(getRunsFromBlock(block['Name']))
            return runs
        return None


    def getDBSSummaryInfo(self, dataset=None, block=None):

        """Dataset summary"""
        def getLumisectionsInBlock(b):
            lumis = set()
            for file in self.dataBlocks.getFiles(b):
                for x in file['LumiList']:
                    lumis.add(x['LumiSectionNumber'])
            return lumis

        result = {}
        if block:
            result['NumberOfEvents'] = sum([x['NumberOfEvents']
                                for x in self.dataBlocks.getFiles(block)])
            result['NumberOfFiles'] = len(self.dataBlocks.getFiles(block))

            result['NumberOfLumis'] = len(getLumisectionsInBlock(block))
#.........这里部分代码省略.........