Python stored_values.StoredValueManager类代码示例

class StoredValueTransform(TransformStreamListener):
    '''
    Receives granules from a stream and persists the latest value in the object store

    Background:
        Platforms publish geospatial information on a separate stream from the 
        instruments. Various data products require geospatial information about
        the instrument to calculate the variables. This component persists the
        latest value in a simple data storage container where complex data
        containers can access it.
    '''
        
    def on_start(self):
        TransformStreamListener.on_start(self)
        self.document_key = self.CFG.get_safe('process.document_key')
        self.stored_value_manager = StoredValueManager(self.container)

    def recv_packet(self, msg, route, stream_id):
        rdt = RecordDictionaryTool.load_from_granule(msg)

        document = {}

        for k,v in rdt.iteritems():
            value_array = np.atleast_1d(v[:])
            if 'f' in value_array.dtype.str:
                document[k] = float(value_array[-1])
            elif 'i' in value_array.dtype.str:
                document[k] = int(value_array[-1])

        self.stored_value_manager.stored_value_cas(self.document_key, document)

    def test_qc_attachment(self):
        instrument_device = InstrumentDevice(name='whatever')
        instrument_device_id,_ = self.rrclient.create(instrument_device)
        self.addCleanup(self.rrclient.delete, instrument_device_id)
        self.client.register_instrument(instrument_device_id)
        self.addCleanup(self.client.unregister_instrument, instrument_device_id)
        dp = DataProduct(name='instrument output')

        dp_id,_ = self.rrclient.create(dp)
        self.addCleanup(self.rrclient.delete, dp_id)

        parser_id = self.make_grt_parser()
        attachment = Attachment(name='qc ref', attachment_type=AttachmentType.REFERENCE,content=global_range_test_document, context=ReferenceAttachmentContext(parser_id=parser_id))
        att_id = self.rrclient.create_attachment(dp_id, attachment)
        self.addCleanup(self.rrclient.delete_attachment, att_id)

        attachment2 = Attachment(name='qc ref2', attachment_type=AttachmentType.REFERENCE, content=global_range_test_document2, context=ReferenceAttachmentContext(parser_id=parser_id))
        att2_id = self.rrclient.create_attachment(dp_id, attachment2)
        self.addCleanup(self.rrclient.delete_attachment, att2_id)

        self.client.assign_data_product(instrument_device_id, dp_id)
        self.addCleanup(self.client.unassign_data_product, instrument_device_id, dp_id)
        svm = StoredValueManager(self.container)
        doc = svm.read_value('grt_CE01ISSM-MF005-01-CTDBPC999_TEMPWAT')
        np.testing.assert_array_almost_equal(doc['grt_min_value'], -2.)

    def fetch_lookup_values(self):
        doc_keys = []
        for lv in self._lookup_values():
            context = self.context(lv)
            if context.document_key:
                document_key = context.document_key
                if "$designator" in context.document_key and "reference_designator" in self._stream_config:
                    document_key = document_key.replace("$designator", self._stream_config["reference_designator"])
                doc_keys.append(document_key)

        lookup_docs = {}
        if doc_keys:
            svm = StoredValueManager(Container.instance)
            doc_list = svm.read_value_mult(doc_keys)
            lookup_docs = dict(zip(doc_keys, doc_list))

        for lv in self._lookup_values():
            context = self.context(lv)
            if context.document_key:
                document_key = context.document_key
                if "$designator" in context.document_key and "reference_designator" in self._stream_config:
                    document_key = document_key.replace("$designator", self._stream_config["reference_designator"])
                doc = lookup_docs[document_key]
                if doc is None:
                    log.debug("Reference Document for %s not found", document_key)
                    continue
                if context.lookup_value in doc:
                    self[lv] = [doc[context.lookup_value]] * self._shp[0] if self._shp else doc[context.lookup_value]

 def parse_document(cls, container, parser, document_path):
     svm = StoredValueManager(container)
     document = ''
     with open(document_path,'r') as f:
         document = f.read()
     for k,v in parser(document):
         svm.stored_value_cas(k,v)
     return

class TestParsers(IonIntegrationTestCase):
    def setUp(self):
        self._start_container()
        self.svm = StoredValueManager(self.container)

    @classmethod
    def parse_document(cls, container, parser, document_path):
        svm = StoredValueManager(container)
        document = ''
        with open(document_path,'r') as f:
            document = f.read()
        for k,v in parser(document):
            svm.stored_value_cas(k,v)
        return


    def test_grt_parser(self):
        self.parse_document(self.container, grt_parser, qc_paths['grt'])

        ret_doc = self.svm.read_value('grt_%s_%s' % ('CE04OSBP-LJ01C-06-CTDBPO108', 'CONDWAT'))
        np.testing.assert_almost_equal(ret_doc['grt_min_value'], 0.)
        np.testing.assert_almost_equal(ret_doc['grt_max_value'], 66000.)

    def test_spike_parser(self):
        self.parse_document(self.container, spike_parser, qc_paths['spike'])

        ret_doc = self.svm.read_value('spike_%s_%s' % ('CE04OSBP-LJ01C-06-CTDBPO108', 'PRACSAL'))
        np.testing.assert_almost_equal(ret_doc['acc'], 0.005)
        np.testing.assert_almost_equal(ret_doc['spike_n'], 11.)
        np.testing.assert_almost_equal(ret_doc['spike_l'], 15.)

        self.assertRaises(NotFound, self.svm.read_value,'spike_%s_%s' % ('CE04OSBP-LJ01C-06-CTDBPO108', 'CLOWN'))


    def test_stuck_value_parser(self):
        self.parse_document(self.container, stuck_value_test_parser, qc_paths['stuck'])

        ret_doc = self.svm.read_value('svt_%s_%s' % ('CE04OSBP-LJ01C-06-CTDBPO108', 'PRACSAL'))
        np.testing.assert_almost_equal(ret_doc['svt_resolution'], 1e-9)
        np.testing.assert_almost_equal(ret_doc['svt_n'], 1000000)

    def test_trend_value_parser(self):
        self.parse_document(self.container, trend_parser, qc_paths['trend'])

        ret_doc = self.svm.read_value('trend_%s_%s' % ('CE04OSBP-LJ01C-06-CTDBPO108', 'PRACSAL'))
        np.testing.assert_almost_equal(ret_doc['time_interval'], 365.0)
        np.testing.assert_almost_equal(ret_doc['polynomial_order'], 1.0)
        np.testing.assert_almost_equal(ret_doc['standard_deviation'], 0.0)

    
    def test_lrt_parser(self):
        self.parse_document(self.container, lrt_parser, qc_paths['lrt'])

        ret_doc = self.svm.read_value('lrt_%s_%s' %('GP02HYPM-SP001-04-CTDPF0999', 'PRACSAL'))

        np.testing.assert_array_equal(ret_doc['datlim'][0], np.array([32.289, 32.927]))
        self.assertEquals(ret_doc['dims'], ['pressure', 'month'])

    def test_global_range_test_parser(self):
        svm = StoredValueManager(self.container)
        for key,doc in grt_parser(grt_sample_doc):
            svm.stored_value_cas(key,doc)
            self.addCleanup(svm.delete_stored_value,key)

        doc = svm.read_value('grt_sbe37-abc123_TEMPWAT_UHHH')
        self.assertEquals(doc['grt_min_value'], 0.)
        self.assertEquals(doc['array'],'array 1')
        doc = svm.read_value('grt_sbe37-abc123_PRESWAT_flagged')
        self.assertEquals(doc['grt_max_value'], 689.47)

 def fetch_lookup_values(self):
     for lv in self._lookup_values():
         context = self.context(lv)
         if context.document_key:
             svm = StoredValueManager(Container.instance)
             try:
                 doc = svm.read_value(context.document_key)
             except NotFound:
                 continue
             if context.lookup_value in doc:
                 self[lv] = doc[context.lookup_value]

    def test_stuck_value_test(self):
        svm = StoredValueManager(self.container)
        for key,doc in stuck_value_test_parser(stuck_value_test_sample_doc):
            svm.stored_value_cas(key,doc)
            self.addCleanup(svm.delete_stored_value,key)

        doc = svm.read_value('svt_ssxbt-ssn719_PRESSURE')
        self.assertEquals(doc['svt_resolution'], 1.2)
        self.assertEquals(doc['svt_n'], 10.)

        doc = svm.read_value('svt_ssxbt-ssn719_COND')
        self.assertEquals(doc['svt_resolution'], 0.2)
        self.assertEquals(doc['units'], 'S m-1')

    def test_stuck_value_test(self):
        svm = StoredValueManager(self.container)
        for key, doc in trend_parser(trend_value_test_sample_doc):
            svm.stored_value_cas(key, doc)
            self.addCleanup(svm.delete_stored_value, key)

        doc = svm.read_value("trend_ssxbt-ssn719_PRESSURE")
        self.assertEquals(doc["time_interval"], 1.0)
        self.assertEquals(doc["standard_deviation"], 0.0)

        doc = svm.read_value("trend_ssxbt-ssn719_COND")
        self.assertEquals(doc["time_interval"], 3.0)
        self.assertEquals(doc["polynomial_order"], "third")

    def test_gradient_test(self):
        svm = StoredValueManager(self.container)
        for key,doc in gradient_test_parser(gradient_test_sample_doc):
            svm.stored_value_cas(key,doc)
            self.addCleanup(svm.delete_stored_value,key)

        doc = svm.read_value('grad_CHEL-C-32-12_PRESSURE_DENSITY')
        self.assertEquals(doc['units_dat'], 'dbar')
        self.assertEquals(doc['d_dat_dx'], 2.781)

        doc = svm.read_value('grad_CHEW-BA-C-A12_PRESSURE_DENSITY')
        self.assertEquals(doc['units_x'], 'kg m-3')
        self.assertEquals(doc['tol_dat'], 12.)

 def fetch_lookup_values(self):
     for lv in self._lookup_values():
         context = self.context(lv)
         if context.document_key:
             document_key = context.document_key
             if '$designator' in context.document_key and 'reference_designator' in self._stream_config:
                 document_key = document_key.replace('$designator',self._stream_config['reference_designator'])
             svm = StoredValueManager(Container.instance)
             try:
                 doc = svm.read_value(document_key)
             except NotFound:
                 log.debug('Reference Document for %s not found', document_key)
                 continue
             if context.lookup_value in doc:
                 self[lv] = [doc[context.lookup_value]] * self._shp[0] if self._shp else doc[context.lookup_value]

class TestParsers(IonIntegrationTestCase):
    def setUp(self):
        self._start_container()
        self.svm = StoredValueManager(self.container)

    @classmethod
    def parse_document(cls, container, parser, document_path):
        svm = StoredValueManager(container)
        document = ''
        with open(document_path,'r') as f:
            document = f.read()
        for k,v in parser(document):
            svm.stored_value_cas(k,v)
        return


    def test_grt_parser(self):
        self.parse_document(self.container, grt_parser, qc_paths['grt'])

        ret_doc = self.svm.read_value('grt_%s_%s' % ('GA01SUMO-RI003-03-CTDMOQ999', 'CONDWAT'))
        np.testing.assert_almost_equal(ret_doc['grt_min_value'], 0.)
        np.testing.assert_almost_equal(ret_doc['grt_max_value'], 66000.)

    def test_spike_parser(self):
        self.parse_document(self.container, spike_parser, qc_paths['spike'])

        ret_doc = self.svm.read_value('spike_%s_%s' % ('GA01SUMO-RI003-03-CTDMOQ999', 'PRACSAL'))
        np.testing.assert_almost_equal(ret_doc['acc'], 0.005)
        np.testing.assert_almost_equal(ret_doc['spike_n'], 11.)
        np.testing.assert_almost_equal(ret_doc['spike_l'], 15.)

        self.assertRaises(NotFound, self.svm.read_value,'spike_%s_%s' % ('GA01SUMO-RI003-03-CTDMOQ999', 'DENSITY'))


    def test_stuck_value_parser(self):
        self.parse_document(self.container, stuck_value_test_parser, qc_paths['stuck'])

        ret_doc = self.svm.read_value('svt_%s_%s' % ('GA01SUMO-RI003-03-CTDMOQ999', 'PRACSAL'))
        np.testing.assert_almost_equal(ret_doc['svt_resolution'], 0.0001)
        np.testing.assert_almost_equal(ret_doc['svt_n'], 20.)

    def test_trend_value_parser(self):
        self.parse_document(self.container, trend_parser, qc_paths['trend'])

        ret_doc = self.svm.read_value('trend_%s_%s' % ('RS01SBVM-LJ01A-05-HPIESA101', 'IESPRES'))
        np.testing.assert_almost_equal(ret_doc['time_interval'], 90.)
        np.testing.assert_almost_equal(ret_doc['polynomial_order'], 1.0)
        np.testing.assert_almost_equal(ret_doc['standard_deviation'], 5.0)

    def test_rdt_lookup(self):
        rdt = self.create_lookup_rdt()

        self.assertTrue('offset_a' in rdt.lookup_values())
        self.assertFalse('offset_b' in rdt.lookup_values())

        rdt['time'] = [0]
        rdt['temp'] = [10.0]
        rdt['offset_a'] = [2.0]
        self.assertEquals(rdt['offset_b'], None)
        self.assertEquals(rdt.lookup_values(), ['offset_a'])
        np.testing.assert_array_almost_equal(rdt['calibrated'], np.array([12.0]))

        svm = StoredValueManager(self.container)
        svm.stored_value_cas('coefficient_document', {'offset_b':2.0})
        rdt.fetch_lookup_values()
        np.testing.assert_array_equal(rdt['offset_b'], np.array([2.0]))
        np.testing.assert_array_equal(rdt['calibrated_b'], np.array([14.0]))

    def setUp(self):
        DMTestCase.setUp(self)
        self.ph = ParameterHelper(self.dataset_management, self.addCleanup)
        pdict_id = self.ph.create_simple_qc_pdict()

        self.stream_def_id = self.pubsub_management.create_stream_definition('global range', parameter_dictionary_id=pdict_id, stream_configuration={'reference_designator':'QCTEST'})
        self.addCleanup(self.pubsub_management.delete_stream_definition, self.stream_def_id)

        self.rdt = RecordDictionaryTool(stream_definition_id=self.stream_def_id)
        self.svm = StoredValueManager(self.container)

 def on_start(self):
     TransformDataProcess.on_start(self)
     self.pubsub_management = PubsubManagementServiceProcessClient(process=self)
     self.stored_values = StoredValueManager(self.container)
     self.input_data_product_ids = self.CFG.get_safe('process.input_products', [])
     self.output_data_product_ids = self.CFG.get_safe('process.output_products', [])
     self.lookup_docs = self.CFG.get_safe('process.lookup_docs',[])
     self.new_lookups = Queue()
     self.lookup_monitor = EventSubscriber(event_type=OT.ExternalReferencesUpdatedEvent,callback=self._add_lookups, auto_delete=True)
     self.lookup_monitor.start()