Python RecordDictionaryTool.iteritems方法代码示例

# 需要导入模块: from ion.services.dm.utility.granule import RecordDictionaryTool [as 别名]
# 或者: from ion.services.dm.utility.granule.RecordDictionaryTool import iteritems [as 别名]
    def test_filter(self):
        pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
        filtered_stream_def_id = self.pubsub_management.create_stream_definition('filtered', parameter_dictionary_id=pdict_id, available_fields=['time', 'temp'])
        self.addCleanup(self.pubsub_management.delete_stream_definition, filtered_stream_def_id)
        rdt = RecordDictionaryTool(stream_definition_id=filtered_stream_def_id)
        self.assertEquals(rdt._available_fields,['time','temp'])
        rdt['time'] = np.arange(20)
        rdt['temp'] = np.arange(20)
        with self.assertRaises(KeyError):
            rdt['pressure'] = np.arange(20)

        granule = rdt.to_granule()
        rdt2 = RecordDictionaryTool.load_from_granule(granule)
        self.assertEquals(rdt._available_fields, rdt2._available_fields)
        self.assertEquals(rdt.fields, rdt2.fields)
        for k,v in rdt.iteritems():
            self.assertTrue(np.array_equal(rdt[k], rdt2[k]))

# 需要导入模块: from ion.services.dm.utility.granule import RecordDictionaryTool [as 别名]
# 或者: from ion.services.dm.utility.granule.RecordDictionaryTool import iteritems [as 别名]
class CoverageCraft(object):
    '''
    AKA the BlackBox
    PFM courtesy of Tommy Vandesteene
    '''
    def __init__(self,coverage=None, granule=None):
        if coverage is None:
            self.coverage = self.create_coverage()
            self.rdt = RecordDictionaryTool(param_dictionary=self.coverage.parameter_dictionary)
        else:
            self.coverage = coverage
            if granule is not None:
                self.sync_with_granule(granule)
            else:
                self.rdt = RecordDictionaryTool(param_dictionary=self.coverage.parameter_dictionary)
        self.pdict = self.coverage.parameter_dictionary


    def sync_rdt_with_granule(self, granule):
        rdt = RecordDictionaryTool.load_from_granule(granule)
        self.rdt = rdt
        return rdt

    def sync_with_granule(self, granule=None):
        if granule is not None:
            self.sync_rdt_with_granule(granule)
        if self.rdt is None:
            log.error('Failed to add granule, no granule assigned.')
            return
        start_index = self.coverage.num_timesteps 
        elements = self.rdt._shp[0]
        if not elements: return
        self.coverage.insert_timesteps(elements)

        for k,v in self.rdt.iteritems():
            log.info("key: %s" , k)
            log.info("value: %s" , v)
            slice_ = slice(start_index,None)
            log.info("slice: %s",  slice_)
            self.coverage.set_parameter_values(param_name=k,tdoa=slice_, value=v)


    def sync_rdt_with_coverage(self, coverage=None, tdoa=None, start_time=None, end_time=None, stride_time=None, parameters=None):
        '''
        Builds a granule based on the coverage
        '''
        if coverage is None:
            coverage = self.coverage

        slice_ = slice(None) # Defaults to all values
        if tdoa is not None and isinstance(tdoa,slice):
            slice_ = tdoa

        elif stride_time is not None:
            validate_is_instance(start_time, Number, 'start_time must be a number for striding.')
            validate_is_instance(end_time, Number, 'end_time must be a number for striding.')
            validate_is_instance(stride_time, Number, 'stride_time must be a number for striding.')
            ugly_range = np.arange(start_time, end_time, stride_time)
            idx_values = [TimeUtils.get_relative_time(coverage,i) for i in ugly_range]
            slice_ = [idx_values]

        elif not (start_time is None and end_time is None):
            time_var = coverage._temporal_param_name
            uom = coverage.get_parameter_context(time_var).uom
            if start_time is not None:
                start_units = TimeUtils.ts_to_units(uom,start_time)
                log.info('Units: %s', start_units)
                start_idx = TimeUtils.get_relative_time(coverage,start_units)
                log.info('Start Index: %s', start_idx)
                start_time = start_idx
            if end_time is not None:
                end_units   = TimeUtils.ts_to_units(uom,end_time)
                log.info('End units: %s', end_units)
                end_idx   = TimeUtils.get_relative_time(coverage,end_units)
                log.info('End index: %s',  end_idx)
                end_time = end_idx
            slice_ = slice(start_time,end_time,stride_time)
            log.info('Slice: %s', slice_)

        if parameters is not None:
            pdict = ParameterDictionary()
            params = set(coverage.list_parameters()).intersection(parameters)
            for param in params:
                pdict.add_context(coverage.get_parameter_context(param))
            rdt = RecordDictionaryTool(param_dictionary=pdict)
            self.pdict = pdict
        else:
            rdt = RecordDictionaryTool(param_dictionary=coverage.parameter_dictionary)
        
        fields = coverage.list_parameters()
        if parameters is not None:
            fields = set(fields).intersection(parameters)

        for d in fields:
            rdt[d] = coverage.get_parameter_values(d,tdoa=slice_)
        self.rdt = rdt # Sync

    def to_granule(self):
        return self.rdt.to_granule()

#.........这里部分代码省略.........

# 需要导入模块: from ion.services.dm.utility.granule import RecordDictionaryTool [as 别名]
# 或者: from ion.services.dm.utility.granule.RecordDictionaryTool import iteritems [as 别名]
    def test_granule(self):
        
        pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
        stream_def_id = self.pubsub_management.create_stream_definition('ctd', parameter_dictionary_id=pdict_id)
        pdict = DatasetManagementService.get_parameter_dictionary_by_name('ctd_parsed_param_dict')
        self.addCleanup(self.pubsub_management.delete_stream_definition,stream_def_id)

        stream_id, route = self.pubsub_management.create_stream('ctd_stream', 'xp1', stream_definition_id=stream_def_id)
        self.addCleanup(self.pubsub_management.delete_stream,stream_id)
        self.xps.append('xp1')
        publisher = StandaloneStreamPublisher(stream_id, route)

        subscriber = StandaloneStreamSubscriber('sub', self.verify_incoming)
        subscriber.start()

        subscription_id = self.pubsub_management.create_subscription('sub', stream_ids=[stream_id])
        self.xns.append('sub')
        self.pubsub_management.activate_subscription(subscription_id)


        rdt = RecordDictionaryTool(stream_definition_id=stream_def_id)
        rdt['time'] = np.arange(10)
        rdt['temp'] = np.random.randn(10) * 10 + 30
        rdt['pressure'] = [20] * 10

        self.assertEquals(set(pdict.keys()), set(rdt.fields))
        self.assertEquals(pdict.temporal_parameter_name, rdt.temporal_parameter)

        self.rdt = rdt
        self.data_producer_id = 'data_producer'
        self.provider_metadata_update = {1:1}

        publisher.publish(rdt.to_granule(data_producer_id='data_producer', provider_metadata_update={1:1}))

        self.assertTrue(self.event.wait(10))
        
        self.pubsub_management.deactivate_subscription(subscription_id)
        self.pubsub_management.delete_subscription(subscription_id)
        
        filtered_stream_def_id = self.pubsub_management.create_stream_definition('filtered', parameter_dictionary_id=pdict_id, available_fields=['time', 'temp'])
        self.addCleanup(self.pubsub_management.delete_stream_definition, filtered_stream_def_id)
        rdt = RecordDictionaryTool(stream_definition_id=filtered_stream_def_id)
        self.assertEquals(rdt._available_fields,['time','temp'])
        rdt['time'] = np.arange(20)
        rdt['temp'] = np.arange(20)
        with self.assertRaises(KeyError):
            rdt['pressure'] = np.arange(20)

        granule = rdt.to_granule()
        rdt2 = RecordDictionaryTool.load_from_granule(granule)
        self.assertEquals(rdt._available_fields, rdt2._available_fields)
        self.assertEquals(rdt.fields, rdt2.fields)
        for k,v in rdt.iteritems():
            self.assertTrue(np.array_equal(rdt[k], rdt2[k]))
        
        rdt = RecordDictionaryTool(stream_definition_id=stream_def_id)
        rdt['time'] = np.array([None,None,None])
        self.assertTrue(rdt['time'] is None)
        
        rdt['time'] = np.array([None, 1, 2])
        self.assertEquals(rdt['time'][0], rdt.fill_value('time'))

# 需要导入模块: from ion.services.dm.utility.granule import RecordDictionaryTool [as 别名]
# 或者: from ion.services.dm.utility.granule.RecordDictionaryTool import iteritems [as 别名]
class CoverageCraft(object):
    '''
    AKA the BlackBox
    PFM courtesy of Tommy Vandesteene
    '''
    def __init__(self,coverage=None, granule=None):
        if coverage is None:
            self.coverage = self.create_coverage()
            self.rdt = RecordDictionaryTool(param_dictionary=self.coverage.parameter_dictionary)
        else:
            self.coverage = coverage
            if granule is not None:
                self.sync_with_granule(granule)
            else:
                self.rdt = RecordDictionaryTool(param_dictionary=self.coverage.parameter_dictionary)
        self.pdict = self.coverage.parameter_dictionary


    def sync_rdt_with_granule(self, granule):
        rdt = RecordDictionaryTool.load_from_granule(granule)
        self.rdt = rdt
        return rdt

    def sync_with_granule(self, granule=None):
        if granule is not None:
            self.sync_rdt_with_granule(granule)
        if self.rdt is None:
            log.error('Failed to add granule, no granule assigned.')
            return
        start_index = self.coverage.num_timesteps 
        elements = self.rdt._shp[0]
        if not elements: return
        self.coverage.insert_timesteps(elements)

        for k,v in self.rdt.iteritems():
            log.info("key: %s" , k)
            log.info("value: %s" , v)
            slice_ = slice(start_index,None)
            log.info("slice: %s",  slice_)
            self.coverage.set_parameter_values(param_name=k,tdoa=slice_, value=v)


    def sync_rdt_with_coverage(self, coverage=None, tdoa=None, start_time=None, end_time=None, stride_time=None, parameters=None):
        '''
        Builds a granule based on the coverage
        '''
        if coverage is None:
            coverage = self.coverage

        slice_ = slice(None) # Defaults to all values
        if tdoa is not None and isinstance(tdoa,slice):
            slice_ = tdoa

        elif stride_time is not None:
            validate_is_instance(start_time, Number, 'start_time must be a number for striding.')
            validate_is_instance(end_time, Number, 'end_time must be a number for striding.')
            validate_is_instance(stride_time, Number, 'stride_time must be a number for striding.')
            ugly_range = np.arange(start_time, end_time, stride_time)
            idx_values = [self.get_relative_time(coverage,i) for i in ugly_range]
            slice_ = [idx_values]

        elif not (start_time is None and end_time is None):
            time_var = coverage._temporal_param_name
            uom = coverage.get_parameter_context(time_var).uom
            if start_time is not None:
                start_units = self.ts_to_units(uom,start_time)
                log.info('Units: %s', start_units)
                start_idx = self.get_relative_time(coverage,start_units)
                log.info('Start Index: %s', start_idx)
                start_time = start_idx
            if end_time is not None:
                end_units   = self.ts_to_units(uom,end_time)
                log.info('End units: %s', end_units)
                end_idx   = self.get_relative_time(coverage,end_units)
                log.info('End index: %s',  end_idx)
                end_time = end_idx
            slice_ = slice(start_time,end_time,stride_time)
            log.info('Slice: %s', slice_)

        if parameters is not None:
            pdict = ParameterDictionary()
            params = set(coverage.list_parameters()).intersection(parameters)
            for param in params:
                pdict.add_context(coverage.get_parameter_context(param))
            rdt = RecordDictionaryTool(param_dictionary=pdict)
            self.pdict = pdict
        else:
            rdt = RecordDictionaryTool(param_dictionary=coverage.parameter_dictionary)
        
        fields = coverage.list_parameters()
        if parameters is not None:
            fields = set(fields).intersection(parameters)

        for d in fields:
            rdt[d] = coverage.get_parameter_values(d,tdoa=slice_)
        self.rdt = rdt # Sync

    def to_granule(self):
        return self.rdt.to_granule()

#.........这里部分代码省略.........