Python StoredValueManager.read_value方法代码示例

# 需要导入模块: from ion.util.stored_values import StoredValueManager [as 别名]
# 或者: from ion.util.stored_values.StoredValueManager import read_value [as 别名]
    def test_qc_attachment(self):
        instrument_device = InstrumentDevice(name='whatever')
        instrument_device_id,_ = self.rrclient.create(instrument_device)
        self.addCleanup(self.rrclient.delete, instrument_device_id)
        self.client.register_instrument(instrument_device_id)
        self.addCleanup(self.client.unregister_instrument, instrument_device_id)
        dp = DataProduct(name='instrument output')

        dp_id,_ = self.rrclient.create(dp)
        self.addCleanup(self.rrclient.delete, dp_id)

        parser_id = self.make_grt_parser()
        attachment = Attachment(name='qc ref', attachment_type=AttachmentType.REFERENCE,content=global_range_test_document, context=ReferenceAttachmentContext(parser_id=parser_id))
        att_id = self.rrclient.create_attachment(dp_id, attachment)
        self.addCleanup(self.rrclient.delete_attachment, att_id)

        attachment2 = Attachment(name='qc ref2', attachment_type=AttachmentType.REFERENCE, content=global_range_test_document2, context=ReferenceAttachmentContext(parser_id=parser_id))
        att2_id = self.rrclient.create_attachment(dp_id, attachment2)
        self.addCleanup(self.rrclient.delete_attachment, att2_id)

        self.client.assign_data_product(instrument_device_id, dp_id)
        self.addCleanup(self.client.unassign_data_product, instrument_device_id, dp_id)
        svm = StoredValueManager(self.container)
        doc = svm.read_value('grt_TEST_TEMPWAT_TEMPWAT')
        np.testing.assert_array_almost_equal(doc['grt_min_value'], 10.)

        doc = svm.read_value('grt_TEST_PRACSAL_PRACSAL')
        np.testing.assert_array_almost_equal(doc['grt_min_value'], 0.)

# 需要导入模块: from ion.util.stored_values import StoredValueManager [as 别名]
# 或者: from ion.util.stored_values.StoredValueManager import read_value [as 别名]
class TestParsers(IonIntegrationTestCase):
    def setUp(self):
        self._start_container()
        self.svm = StoredValueManager(self.container)

    @classmethod
    def parse_document(cls, container, parser, document_path):
        svm = StoredValueManager(container)
        document = ''
        with open(document_path,'r') as f:
            document = f.read()
        for k,v in parser(document):
            svm.stored_value_cas(k,v)
        return


    def test_grt_parser(self):
        self.parse_document(self.container, grt_parser, qc_paths['grt'])

        ret_doc = self.svm.read_value('grt_%s_%s' % ('CE04OSBP-LJ01C-06-CTDBPO108', 'CONDWAT'))
        np.testing.assert_almost_equal(ret_doc['grt_min_value'], 0.)
        np.testing.assert_almost_equal(ret_doc['grt_max_value'], 66000.)

    def test_spike_parser(self):
        self.parse_document(self.container, spike_parser, qc_paths['spike'])

        ret_doc = self.svm.read_value('spike_%s_%s' % ('CE04OSBP-LJ01C-06-CTDBPO108', 'PRACSAL'))
        np.testing.assert_almost_equal(ret_doc['acc'], 0.005)
        np.testing.assert_almost_equal(ret_doc['spike_n'], 11.)
        np.testing.assert_almost_equal(ret_doc['spike_l'], 15.)

        self.assertRaises(NotFound, self.svm.read_value,'spike_%s_%s' % ('CE04OSBP-LJ01C-06-CTDBPO108', 'CLOWN'))


    def test_stuck_value_parser(self):
        self.parse_document(self.container, stuck_value_test_parser, qc_paths['stuck'])

        ret_doc = self.svm.read_value('svt_%s_%s' % ('CE04OSBP-LJ01C-06-CTDBPO108', 'PRACSAL'))
        np.testing.assert_almost_equal(ret_doc['svt_resolution'], 1e-9)
        np.testing.assert_almost_equal(ret_doc['svt_n'], 1000000)

    def test_trend_value_parser(self):
        self.parse_document(self.container, trend_parser, qc_paths['trend'])

        ret_doc = self.svm.read_value('trend_%s_%s' % ('CE04OSBP-LJ01C-06-CTDBPO108', 'PRACSAL'))
        np.testing.assert_almost_equal(ret_doc['time_interval'], 365.0)
        np.testing.assert_almost_equal(ret_doc['polynomial_order'], 1.0)
        np.testing.assert_almost_equal(ret_doc['standard_deviation'], 0.0)

    
    def test_lrt_parser(self):
        self.parse_document(self.container, lrt_parser, qc_paths['lrt'])

        ret_doc = self.svm.read_value('lrt_%s_%s' %('GP02HYPM-SP001-04-CTDPF0999', 'PRACSAL'))

        np.testing.assert_array_equal(ret_doc['datlim'][0], np.array([32.289, 32.927]))
        self.assertEquals(ret_doc['dims'], ['pressure', 'month'])

# 需要导入模块: from ion.util.stored_values import StoredValueManager [as 别名]
# 或者: from ion.util.stored_values.StoredValueManager import read_value [as 别名]
    def test_global_range_test_parser(self):
        svm = StoredValueManager(self.container)
        for key,doc in grt_parser(grt_sample_doc):
            svm.stored_value_cas(key,doc)
            self.addCleanup(svm.delete_stored_value,key)

        doc = svm.read_value('grt_sbe37-abc123_TEMPWAT_UHHH')
        self.assertEquals(doc['grt_min_value'], 0.)
        self.assertEquals(doc['array'],'array 1')
        doc = svm.read_value('grt_sbe37-abc123_PRESWAT_flagged')
        self.assertEquals(doc['grt_max_value'], 689.47)

# 需要导入模块: from ion.util.stored_values import StoredValueManager [as 别名]
# 或者: from ion.util.stored_values.StoredValueManager import read_value [as 别名]
    def test_stuck_value_test(self):
        svm = StoredValueManager(self.container)
        for key, doc in trend_parser(trend_value_test_sample_doc):
            svm.stored_value_cas(key, doc)
            self.addCleanup(svm.delete_stored_value, key)

        doc = svm.read_value("trend_ssxbt-ssn719_PRESSURE")
        self.assertEquals(doc["time_interval"], 1.0)
        self.assertEquals(doc["standard_deviation"], 0.0)

        doc = svm.read_value("trend_ssxbt-ssn719_COND")
        self.assertEquals(doc["time_interval"], 3.0)
        self.assertEquals(doc["polynomial_order"], "third")

# 需要导入模块: from ion.util.stored_values import StoredValueManager [as 别名]
# 或者: from ion.util.stored_values.StoredValueManager import read_value [as 别名]
    def test_gradient_test(self):
        svm = StoredValueManager(self.container)
        for key,doc in gradient_test_parser(gradient_test_sample_doc):
            svm.stored_value_cas(key,doc)
            self.addCleanup(svm.delete_stored_value,key)

        doc = svm.read_value('grad_CHEL-C-32-12_PRESSURE_DENSITY')
        self.assertEquals(doc['units_dat'], 'dbar')
        self.assertEquals(doc['d_dat_dx'], 2.781)

        doc = svm.read_value('grad_CHEW-BA-C-A12_PRESSURE_DENSITY')
        self.assertEquals(doc['units_x'], 'kg m-3')
        self.assertEquals(doc['tol_dat'], 12.)

# 需要导入模块: from ion.util.stored_values import StoredValueManager [as 别名]
# 或者: from ion.util.stored_values.StoredValueManager import read_value [as 别名]
    def test_stuck_value_test(self):
        svm = StoredValueManager(self.container)
        for key,doc in stuck_value_test_parser(stuck_value_test_sample_doc):
            svm.stored_value_cas(key,doc)
            self.addCleanup(svm.delete_stored_value,key)

        doc = svm.read_value('svt_ssxbt-ssn719_PRESSURE')
        self.assertEquals(doc['svt_resolution'], 1.2)
        self.assertEquals(doc['svt_n'], 10.)

        doc = svm.read_value('svt_ssxbt-ssn719_COND')
        self.assertEquals(doc['svt_resolution'], 0.2)
        self.assertEquals(doc['units'], 'S m-1')

# 需要导入模块: from ion.util.stored_values import StoredValueManager [as 别名]
# 或者: from ion.util.stored_values.StoredValueManager import read_value [as 别名]
    def test_basic_stored_docs(self):

        doc = {'key':'value'}
        doc_key = 'doc_id'

        svm = StoredValueManager(self.container)
        doc_id, rev = svm.stored_value_cas(doc_key, doc)
        self.addCleanup(svm.delete_stored_value, doc_key)

        doc = svm.read_value(doc_key)
        self.assertTrue('key' in doc and doc['key']=='value')
        svm.stored_value_cas(doc_key,{'key2':'value2'})
        doc = svm.read_value(doc_key)
        self.assertTrue('key' in doc and doc['key']=='value')
        self.assertTrue('key2' in doc and doc['key2']=='value2')

# 需要导入模块: from ion.util.stored_values import StoredValueManager [as 别名]
# 或者: from ion.util.stored_values.StoredValueManager import read_value [as 别名]
class TestParsers(IonIntegrationTestCase):
    def setUp(self):
        self._start_container()
        self.svm = StoredValueManager(self.container)

    @classmethod
    def parse_document(cls, container, parser, document_path):
        svm = StoredValueManager(container)
        document = ''
        with open(document_path,'r') as f:
            document = f.read()
        for k,v in parser(document):
            svm.stored_value_cas(k,v)
        return


    def test_grt_parser(self):
        self.parse_document(self.container, grt_parser, qc_paths['grt'])

        ret_doc = self.svm.read_value('grt_%s_%s' % ('GA01SUMO-RI003-03-CTDMOQ999', 'CONDWAT'))
        np.testing.assert_almost_equal(ret_doc['grt_min_value'], 0.)
        np.testing.assert_almost_equal(ret_doc['grt_max_value'], 66000.)

    def test_spike_parser(self):
        self.parse_document(self.container, spike_parser, qc_paths['spike'])

        ret_doc = self.svm.read_value('spike_%s_%s' % ('GA01SUMO-RI003-03-CTDMOQ999', 'PRACSAL'))
        np.testing.assert_almost_equal(ret_doc['acc'], 0.005)
        np.testing.assert_almost_equal(ret_doc['spike_n'], 11.)
        np.testing.assert_almost_equal(ret_doc['spike_l'], 15.)

        self.assertRaises(NotFound, self.svm.read_value,'spike_%s_%s' % ('GA01SUMO-RI003-03-CTDMOQ999', 'DENSITY'))


    def test_stuck_value_parser(self):
        self.parse_document(self.container, stuck_value_test_parser, qc_paths['stuck'])

        ret_doc = self.svm.read_value('svt_%s_%s' % ('GA01SUMO-RI003-03-CTDMOQ999', 'PRACSAL'))
        np.testing.assert_almost_equal(ret_doc['svt_resolution'], 0.0001)
        np.testing.assert_almost_equal(ret_doc['svt_n'], 20.)

    def test_trend_value_parser(self):
        self.parse_document(self.container, trend_parser, qc_paths['trend'])

        ret_doc = self.svm.read_value('trend_%s_%s' % ('RS01SBVM-LJ01A-05-HPIESA101', 'IESPRES'))
        np.testing.assert_almost_equal(ret_doc['time_interval'], 90.)
        np.testing.assert_almost_equal(ret_doc['polynomial_order'], 1.0)
        np.testing.assert_almost_equal(ret_doc['standard_deviation'], 5.0)

# 需要导入模块: from ion.util.stored_values import StoredValueManager [as 别名]
# 或者: from ion.util.stored_values.StoredValueManager import read_value [as 别名]
 def fetch_lookup_values(self):
     for lv in self._lookup_values():
         context = self.context(lv)
         if context.document_key:
             svm = StoredValueManager(Container.instance)
             try:
                 doc = svm.read_value(context.document_key)
             except NotFound:
                 continue
             if context.lookup_value in doc:
                 self[lv] = doc[context.lookup_value]

# 需要导入模块: from ion.util.stored_values import StoredValueManager [as 别名]
# 或者: from ion.util.stored_values.StoredValueManager import read_value [as 别名]
 def fetch_lookup_values(self):
     for lv in self._lookup_values():
         context = self.context(lv)
         if context.document_key:
             document_key = context.document_key
             if '$designator' in context.document_key and 'reference_designator' in self._stream_config:
                 document_key = document_key.replace('$designator',self._stream_config['reference_designator'])
             svm = StoredValueManager(Container.instance)
             try:
                 doc = svm.read_value(document_key)
             except NotFound:
                 log.debug('Reference Document for %s not found', document_key)
                 continue
             if context.lookup_value in doc:
                 self[lv] = [doc[context.lookup_value]] * self._shp[0] if self._shp else doc[context.lookup_value]

# 需要导入模块: from ion.util.stored_values import StoredValueManager [as 别名]
# 或者: from ion.util.stored_values.StoredValueManager import read_value [as 别名]

#.........这里部分代码省略.........
                    log.error("We're giving up, the coverage needs to be inspected %s", DatasetManagementService._get_coverage_path(dataset_id))
                    raise

                if stream_id in self._coverages:
                    log.info('Popping coverage for stream %s', stream_id)
                    self._coverages.pop(stream_id)

                gevent.sleep(timeout)
                if timeout > (60 * 5):
                    timeout = 60 * 5
                else:
                    timeout *= 2


    def expand_coverage(self, coverage, elements, stream_id):
        try:
            coverage.insert_timesteps(elements, oob=False)
        except IOError as e:
            log.error("Couldn't insert time steps for coverage: %s",
                      coverage.persistence_dir, exc_info=True)
            try:
                coverage.close()
            finally:
                self._bad_coverages[stream_id] = 1
                raise CorruptionError(e.message)
    
    def get_stored_values(self, lookup_value):
        if not self.new_lookups.empty():
            new_values = self.new_lookups.get()
            self.lookup_docs = new_values + self.lookup_docs
        lookup_value_document_keys = self.lookup_docs
        for key in lookup_value_document_keys:
            try:
                document = self.stored_value_manager.read_value(key)
                if lookup_value in document:
                    return document[lookup_value] 
            except NotFound:
                log.warning('Specified lookup document does not exist')
        return None


    def fill_lookup_values(self, rdt):
        rdt.fetch_lookup_values()
        for field in rdt.lookup_values():
            value = self.get_stored_values(rdt.context(field).lookup_value)
            if value:
                rdt[field] = value

    def insert_sparse_values(self, coverage, rdt, stream_id):

        self.fill_lookup_values(rdt)
        for field in rdt._lookup_values():
            if rdt[field] is None:
                continue
            if not isinstance(rdt.context(field).param_type, SparseConstantType):
                # We only set sparse values before insert
                continue 
            value = rdt[field]
            try:
                coverage.set_parameter_values(param_name=field, value=value)
            except IOError as e:
                log.error("Couldn't insert values for coverage: %s",
                          coverage.persistence_dir, exc_info=True)
                try:
                    coverage.close()
                finally:

# 需要导入模块: from ion.util.stored_values import StoredValueManager [as 别名]
# 或者: from ion.util.stored_values.StoredValueManager import read_value [as 别名]

#.........这里部分代码省略.........
                    log.error("We're giving up, the coverage needs to be inspected %s", DatasetManagementService._get_coverage_path(dataset_id))
                    raise

                if stream_id in self._coverages:
                    log.info('Popping coverage for stream %s', stream_id)
                    self._coverages.pop(stream_id)

                gevent.sleep(timeout)
                if timeout > (60 * 5):
                    timeout = 60 * 5
                else:
                    timeout *= 2


    def expand_coverage(self, coverage, elements, stream_id):
        try:
            coverage.insert_timesteps(elements, oob=False)
        except IOError as e:
            log.error("Couldn't insert time steps for coverage: %s",
                      coverage.persistence_dir, exc_info=True)
            try:
                coverage.close()
            finally:
                self._bad_coverages[stream_id] = 1
                raise CorruptionError(e.message)
    
    def get_stored_values(self, lookup_value):
        if not self.new_lookups.empty():
            new_values = self.new_lookups.get()
            self.lookup_docs = new_values + self.lookup_docs
        lookup_value_document_keys = self.lookup_docs
        for key in lookup_value_document_keys:
            try:
                document = self.stored_value_manager.read_value(key)
                if lookup_value in document:
                    return document[lookup_value] 
            except NotFound:
                log.warning('Specified lookup document does not exist')
        return None


    def fill_lookup_values(self, rdt):
        rdt.fetch_lookup_values()
        for field in rdt.lookup_values():
            value = self.get_stored_values(rdt.context(field).lookup_value)
            if value:
                rdt[field] = value

    def insert_sparse_values(self, coverage, rdt, stream_id):

        self.fill_lookup_values(rdt)
        for field in rdt.fields:
            if rdt._rd[field] is None:
                continue
            if not isinstance(rdt.context(field).param_type, SparseConstantType):
                # We only set sparse values before insert
                continue 
            value = rdt[field]
            try:
                coverage.set_parameter_values(param_name=field, value=value)
            except ValueError as e:
                if "'lower_bound' cannot be >= 'upper_bound'" in e.message:
                    continue
                else:
                    raise
            except IOError as e:

# 需要导入模块: from ion.util.stored_values import StoredValueManager [as 别名]
# 或者: from ion.util.stored_values.StoredValueManager import read_value [as 别名]

#.........这里部分代码省略.........
        except AttributeError:
            log.exception('Module %s does not have class %s', repr(module), actor['class'])
            raise
        try:
            execute = getattr(cls,'execute')
        except AttributeError:
            log.exception('Actor class does not contain execute method')
            raise
        return execute

   
    def _execute_actor(self, msg, actor, streams):
        stream_in_id,stream_out_id = streams
        stream_def_out = self.read_stream_def(stream_out_id)
        params = self.CFG.get_safe('process.params', {})
        config = self.CFG.get_safe('process')
        #do the stuff with the actor
        params['stream_def'] = stream_def_out._id
        executor = self._load_actor(actor)
        try:
            rdt_out = executor(msg, None, config, params, None)
        except:
            log.exception('Error running actor for %s', self.id)
            raise
        return rdt_out

    def _merge_pdicts(self, pdict1, pdict2):
        incoming_pdict = ParameterDictionary.load(pdict1)
        outgoing_pdict = ParameterDictionary.load(pdict2)
        
        merged_pdict = ParameterDictionary()
        for k,v in incoming_pdict.iteritems():
            ordinal, v = v
            if k not in merged_pdict:
                merged_pdict.add_context(v)
        for k,v in outgoing_pdict.iteritems():
            ordinal, v = v
            if k not in merged_pdict:
                merged_pdict.add_context(v)
        return merged_pdict

    def _merge_rdt(self, stream_def_in, stream_def_out):
        incoming_pdict_dump = stream_def_in.parameter_dictionary
        outgoing_pdict_dump = stream_def_out.parameter_dictionary

        merged_pdict = self._merge_pdicts(incoming_pdict_dump, outgoing_pdict_dump)
        rdt_temp = RecordDictionaryTool(param_dictionary=merged_pdict)
        return rdt_temp


    def _get_lookup_value(self, lookup_value):
        if not self.new_lookups.empty():
            new_values = self.new_lookups.get()
            self.lookup_docs = new_values + self.lookup_docs

        lookup_value_document_keys = self.lookup_docs
        for key in lookup_value_document_keys:
            try:
                document = self.stored_values.read_value(key)
                if lookup_value in document:
                    return document[lookup_value]
            except NotFound:
                log.warning('Specified lookup document does not exist')

        return None

    def _execute_transform(self, msg, streams):
        stream_in_id,stream_out_id = streams
        stream_def_in = self.read_stream_def(stream_in_id)
        stream_def_out = self.read_stream_def(stream_out_id)

        rdt_temp = self._merge_rdt(stream_def_in, stream_def_out)
        
        rdt_in = RecordDictionaryTool.load_from_granule(msg)
        for field in rdt_temp.fields:
            if not isinstance(rdt_temp._pdict.get_context(field).param_type, ParameterFunctionType):
                try:
                    rdt_temp[field] = rdt_in[field]
                except KeyError:
                    pass

        rdt_temp.fetch_lookup_values()

        for lookup_field in rdt_temp.lookup_values():
            s = lookup_field
            stored_value = self._get_lookup_value(rdt_temp.context(s).lookup_value)
            if stored_value is not None:
                rdt_temp[s] = stored_value
        
        for field in rdt_temp.fields:
            if isinstance(rdt_temp._pdict.get_context(field).param_type, ParameterFunctionType):
                rdt_temp[field] = rdt_temp[field]

        
        rdt_out = RecordDictionaryTool(stream_definition_id=stream_def_out._id)

        for field in rdt_out.fields:
            rdt_out[field] = rdt_temp[field]
        
        return rdt_out