Python DostaAbcdjmCsppParser.get_records方法代码示例

# 需要导入模块: from mi.dataset.parser.dosta_abcdjm_cspp import DostaAbcdjmCsppParser [as 别名]
# 或者: from mi.dataset.parser.dosta_abcdjm_cspp.DostaAbcdjmCsppParser import get_records [as 别名]
    def test_bad_header_start_date(self):
        """
        Ensure that bad data is skipped when it exists.
        """

        with open(os.path.join(RESOURCE_PATH, 'BadHeaderProcessedData_PPB_OPT.txt'), 'r') as stream_handle:

            parser = DostaAbcdjmCsppParser(self.config_recovered, stream_handle, self.exception_callback)

            # parser should return metadata without start date filled in
            parser.get_records(1)
            self.assertEqual(self.exception_callback_value, [])

# 需要导入模块: from mi.dataset.parser.dosta_abcdjm_cspp import DostaAbcdjmCsppParser [as 别名]
# 或者: from mi.dataset.parser.dosta_abcdjm_cspp.DostaAbcdjmCsppParser import get_records [as 别名]
    def test_bad_header_source_file_name(self):
        """
        Ensure that bad source file name produces an error
        """

        with open(os.path.join(RESOURCE_PATH, 'BadHeaderSourceFileName_PPB_OPT.txt'), 'r') as stream_handle:

            parser = DostaAbcdjmCsppParser(self.config_recovered, stream_handle, self.exception_callback)

            parser.get_records(1)

            self.assertEquals(len(self.exception_callback_value), 1)
            self.assertIsInstance(self.exception_callback_value[0], SampleEncodingException)

# 需要导入模块: from mi.dataset.parser.dosta_abcdjm_cspp import DostaAbcdjmCsppParser [as 别名]
# 或者: from mi.dataset.parser.dosta_abcdjm_cspp.DostaAbcdjmCsppParser import get_records [as 别名]
    def test_get_many(self):
        """
        Read test data and pull out multiple data particles at one time.
        Assert that the results are those we expected.
        """

        file_path = os.path.join(RESOURCE_PATH, '11079894_PPB_OPT.txt')
        stream_handle = open(file_path, 'rb')

        parser = DostaAbcdjmCsppParser(self.config.get(DataTypeKey.DOSTA_ABCDJM_CSPP_RECOVERED),
                                       None, stream_handle,
                                       self.state_callback, self.pub_callback,
                                       self.exception_callback)

        # Let's attempt to retrieve 20 particles
        particles = parser.get_records(20)

        log.info("Exception callback value: %s", self.exception_callback_value)

        # Should end up with 20 particles
        self.assertTrue(len(particles) == 20)

        expected_results = self.get_dict_from_yml('11079894_PPB_OPT.yml')
        for i in range(len(particles)):
            self.assert_result(expected_results['data'][i], particles[i])

# 需要导入模块: from mi.dataset.parser.dosta_abcdjm_cspp import DostaAbcdjmCsppParser [as 别名]
# 或者: from mi.dataset.parser.dosta_abcdjm_cspp.DostaAbcdjmCsppParser import get_records [as 别名]
    def test_simple(self):
        """
        Read test data and pull out data particles.
        Assert that the results are those we expected.
        """
        file_path = os.path.join(RESOURCE_PATH, '11079894_PPB_OPT.txt')
        stream_handle = open(file_path, 'rb')

        parser = DostaAbcdjmCsppParser(self.config.get(DataTypeKey.DOSTA_ABCDJM_CSPP_RECOVERED),
                                       None, stream_handle,
                                       self.state_callback, self.pub_callback,
                                       self.exception_callback)

        particles = parser.get_records(5)

        log.info("Exception callback value: %s", self.exception_callback_value)

        self.assertTrue(self.exception_callback_value is None)

        self.assertTrue(len(particles) == 5)

        expected_results = self.get_dict_from_yml('11079894_PPB_OPT.yml')

        for i in range(len(particles)):
            self.assert_result(expected_results['data'][i], particles[i])

        stream_handle.close()

# 需要导入模块: from mi.dataset.parser.dosta_abcdjm_cspp import DostaAbcdjmCsppParser [as 别名]
# 或者: from mi.dataset.parser.dosta_abcdjm_cspp.DostaAbcdjmCsppParser import get_records [as 别名]
    def test_midstate_start(self):
        """
        This test makes sure that we retrieve the correct particles upon starting with an offsetted state.
        """
        expected_results = self.get_dict_from_yml('11079894_PPB_OPT.yml')

        file_path = os.path.join(RESOURCE_PATH, '11079894_PPB_OPT.txt')
        stream_handle = open(file_path, 'rb')

        initial_state = {StateKey.POSITION: 332, StateKey.METADATA_EXTRACTED: True}

        parser = DostaAbcdjmCsppParser(self.config.get(DataTypeKey.DOSTA_ABCDJM_CSPP_RECOVERED),
                                       initial_state, stream_handle,
                                       self.state_callback, self.pub_callback,
                                       self.exception_callback)

        particles = parser.get_records(2)

        log.info("Num particles: %s", len(particles))

        self.assertTrue(len(particles) == 2)

        for i in range(len(particles)):
            self.assert_result(expected_results['data'][i+2], particles[i])

        log.info("******** Read State: %s", parser._read_state)
        log.info("******** State: %s", parser._state)

        self.assertTrue(parser._state == {StateKey.POSITION: 518, StateKey.METADATA_EXTRACTED: True})

# 需要导入模块: from mi.dataset.parser.dosta_abcdjm_cspp import DostaAbcdjmCsppParser [as 别名]
# 或者: from mi.dataset.parser.dosta_abcdjm_cspp.DostaAbcdjmCsppParser import get_records [as 别名]
    def test_bad_header_start_date(self):
        """
        Ensure that bad data is skipped when it exists.
        """

        file_path = os.path.join(RESOURCE_PATH, 'BadHeaderProcessedData_PPB_OPT.txt')
        stream_handle = open(file_path, 'rb')

        log.info(self.exception_callback_value)

        parser = DostaAbcdjmCsppParser(self.config.get(DataTypeKey.DOSTA_ABCDJM_CSPP_RECOVERED),
                                       None, stream_handle,
                                       self.state_callback, self.pub_callback,
                                       self.exception_callback)

        with self.assertRaises(SampleException):
            parser.get_records(1)

        stream_handle.close()

# 需要导入模块: from mi.dataset.parser.dosta_abcdjm_cspp import DostaAbcdjmCsppParser [as 别名]
# 或者: from mi.dataset.parser.dosta_abcdjm_cspp.DostaAbcdjmCsppParser import get_records [as 别名]
    def test_air_saturation_preset(self):
        """
        Ensure that input files containing the air saturation field are parsed correctly.
        Redmine #10238 Identified additional parameter enabled after first deployment
        """
        with open(os.path.join(RESOURCE_PATH, 'ucspp_32260420_PPB_OPT.txt'), 'rU') as stream_handle:

            parser = DostaAbcdjmCsppParser(self.config_recovered, stream_handle, self.exception_callback)

            # get the metadata particle and first 2 instrument particles and verify values.
            particles = parser.get_records(3)

            self.assertTrue(len(particles) == 3)
            self.assertEqual(self.exception_callback_value, [])

            self.assert_particles(particles, 'ucspp_32260420_PPB_OPT.yml', RESOURCE_PATH)

            # get remaining particles and verify parsed without error.
            particles = parser.get_records(100)

            self.assertTrue(len(particles) == 93)
            self.assertEqual(self.exception_callback_value, [])

# 需要导入模块: from mi.dataset.parser.dosta_abcdjm_cspp import DostaAbcdjmCsppParser [as 别名]
# 或者: from mi.dataset.parser.dosta_abcdjm_cspp.DostaAbcdjmCsppParser import get_records [as 别名]
    def test_bad_header_source_file_name(self):
        """
        Ensure that bad data is skipped when it exists.
        """

        file_path = os.path.join(RESOURCE_PATH, 'BadHeaderSourceFileName_PPB_OPT.txt')
        stream_handle = open(file_path, 'rb')

        log.info(self.exception_callback_value)

        parser = DostaAbcdjmCsppParser(self.config.get(DataTypeKey.DOSTA_ABCDJM_CSPP_RECOVERED),
                                       None, stream_handle,
                                       self.state_callback, self.pub_callback,
                                       self.exception_callback)

        parser.get_records(1)

        log.info("Exception callback value: %s", self.exception_callback_value)

        self.assertTrue(self.exception_callback_value != None)

        stream_handle.close()

# 需要导入模块: from mi.dataset.parser.dosta_abcdjm_cspp import DostaAbcdjmCsppParser [as 别名]
# 或者: from mi.dataset.parser.dosta_abcdjm_cspp.DostaAbcdjmCsppParser import get_records [as 别名]
    def test_get_many(self):
        """
        Read test data and pull out data particles in smaller groups.
        Assert that the results are those we expected.
        """

        with open(os.path.join(RESOURCE_PATH, '11079419_PPB_OPT.txt'), 'r') as stream_handle:

            parser = DostaAbcdjmCsppParser(self.config_recovered, stream_handle, self.exception_callback)

            # Attempt to retrieve 20 total particles
            particles = parser.get_records(5)
            particles2 = parser.get_records(10)
            particles.extend(particles2)
            particles3 = parser.get_records(5)
            particles.extend(particles3)

            # Should end up with 20 particles
            self.assertTrue(len(particles) == 20)

            self.assert_particles(particles, '11079419_PPB_OPT.yml', RESOURCE_PATH)

            self.assertEquals(self.exception_callback_value, [])

# 需要导入模块: from mi.dataset.parser.dosta_abcdjm_cspp import DostaAbcdjmCsppParser [as 别名]
# 或者: from mi.dataset.parser.dosta_abcdjm_cspp.DostaAbcdjmCsppParser import get_records [as 别名]
    def test_linux_source_path_handling(self):
        """
        Read linux source path test data and assert that the results are those we expected.
        """
        with open(os.path.join(RESOURCE_PATH, 'linux_11079894_PPB_OPT.txt'), 'r') as stream_handle:

            parser = DostaAbcdjmCsppParser(self.config_recovered, stream_handle, self.exception_callback)

            particles = parser.get_records(20)

            self.assertTrue(len(particles) == 20)

            self.assert_particles(particles, 'linux.yml', RESOURCE_PATH)

            self.assertEqual(self.exception_callback_value, [])

# 需要导入模块: from mi.dataset.parser.dosta_abcdjm_cspp import DostaAbcdjmCsppParser [as 别名]
# 或者: from mi.dataset.parser.dosta_abcdjm_cspp.DostaAbcdjmCsppParser import get_records [as 别名]
    def test_bad_data_record(self):
        """
        Ensure that bad data creates a recoverable sample exception and parsing continues
        """

        with open(os.path.join(RESOURCE_PATH, 'BadDataRecord_PPB_OPT.txt'), 'r') as stream_handle:

            parser = DostaAbcdjmCsppParser(self.config_recovered, stream_handle, self.exception_callback)

            particles = parser.get_records(19)

            self.assert_particles(particles, 'BadDataRecord_PPB_OPT.yml', RESOURCE_PATH)

            self.assertEquals(len(self.exception_callback_value), 1)
            self.assertIsInstance(self.exception_callback_value[0], RecoverableSampleException)

# 需要导入模块: from mi.dataset.parser.dosta_abcdjm_cspp import DostaAbcdjmCsppParser [as 别名]
# 或者: from mi.dataset.parser.dosta_abcdjm_cspp.DostaAbcdjmCsppParser import get_records [as 别名]
    def test_long_stream(self):
        """
        Test a long stream 
        """
        with open(os.path.join(RESOURCE_PATH, '11079364_PPB_OPT.txt'), 'r') as stream_handle:

            parser = DostaAbcdjmCsppParser(self.config_recovered, stream_handle, self.exception_callback)

            # Let's attempt to retrieve more particles than are available
            particles = parser.get_records(300)

            # Should end up with 272 particles
            self.assertTrue(len(particles) == 272)

            self.assertEquals(self.exception_callback_value, [])

# 需要导入模块: from mi.dataset.parser.dosta_abcdjm_cspp import DostaAbcdjmCsppParser [as 别名]
# 或者: from mi.dataset.parser.dosta_abcdjm_cspp.DostaAbcdjmCsppParser import get_records [as 别名]
    def test_simple_recov(self):
        """
        Read test data and pull out data particles.
        Assert that the results are those we expected.
        """

        with open(os.path.join(RESOURCE_PATH, '11079894_PPB_OPT.txt'), 'r') as stream_handle:

            parser = DostaAbcdjmCsppParser(self.config_recovered, stream_handle, self.exception_callback)

            # Attempt to retrieve 20 particles, there are more in this file but only verify 20
            particles = parser.get_records(20)

            # Should end up with 20 particles
            self.assertTrue(len(particles) == 20)

            self.assert_particles(particles, '11079894_PPB_OPT.yml', RESOURCE_PATH)

            self.assertEquals(self.exception_callback_value, [])

# 需要导入模块: from mi.dataset.parser.dosta_abcdjm_cspp import DostaAbcdjmCsppParser [as 别名]
# 或者: from mi.dataset.parser.dosta_abcdjm_cspp.DostaAbcdjmCsppParser import get_records [as 别名]
    def test_simple_telem(self):
        """
        Read test data and pull out data particles.
        Assert that the results are those we expected.
        """

        with open(os.path.join(RESOURCE_PATH, '11194982_PPD_OPT.txt'), 'r') as stream_handle:

            parser = DostaAbcdjmCsppParser(self.config_telemetered, stream_handle, self.exception_callback)

            # Attempt to retrieve 20 particles, there are only 18 in the file though
            particles = parser.get_records(20)

            # Should end up with 18 particles
            self.assertTrue(len(particles) == 18)

            self.assert_particles(particles, '11194982_PPD_OPT.yml', RESOURCE_PATH)

            self.assertEquals(self.exception_callback_value, [])

# 需要导入模块: from mi.dataset.parser.dosta_abcdjm_cspp import DostaAbcdjmCsppParser [as 别名]
# 或者: from mi.dataset.parser.dosta_abcdjm_cspp.DostaAbcdjmCsppParser import get_records [as 别名]
    def test_long_stream(self):
        """
        Test a long stream 
        """

        file_path = os.path.join(RESOURCE_PATH, '11079364_PPB_OPT.txt')
        stream_handle = open(file_path, 'rb')

        parser = DostaAbcdjmCsppParser(self.config.get(DataTypeKey.DOSTA_ABCDJM_CSPP_RECOVERED),
                                       None, stream_handle,
                                       self.state_callback, self.pub_callback,
                                       self.exception_callback)

        # Let's attempt to retrieve 2000 particles
        particles = parser.get_records(300)

        log.info("Num particles: %s", len(particles))

        log.info("Exception callback value: %s", self.exception_callback_value)

        # Should end up with 272 particles
        self.assertTrue(len(particles) == 272)

        stream_handle.close()