Python singer.parse_message方法代码示例

# 需要导入模块: import singer [as 别名]
# 或者: from singer import parse_message [as 别名]
def test_round_trip(self):
        record_message = singer.RecordMessage(
            record={'name': 'foo'},
            stream='users')

        schema_message = singer.SchemaMessage(
            stream='users',
            key_properties=['name'],
            schema={'type': 'object',
                    'properties': {
                        'name': {'type': 'string'}}})

        state_message = singer.StateMessage(value={'seq': 1})

        self.assertEqual(record_message,
                         singer.parse_message(singer.format_message(record_message)))
        self.assertEqual(schema_message,
                         singer.parse_message(singer.format_message(schema_message)))
        self.assertEqual(state_message,
                         singer.parse_message(singer.format_message(state_message)))

    ## These three tests just confirm that writing doesn't throw 

# 需要导入模块: import singer [as 别名]
# 或者: from singer import parse_message [as 别名]
def summarize_output(output):
    summary = OutputSummary()
    for line in output:
        summary.add(singer.parse_message(line))
    return summary 

# 需要导入模块: import singer [as 别名]
# 或者: from singer import parse_message [as 别名]
def test_parse_message_record_good(self):
        message = singer.parse_message(
            '{"type": "RECORD", "record": {"name": "foo"}, "stream": "users"}')
        self.assertEqual(
            message,
            singer.RecordMessage(record={'name': 'foo'}, stream='users')) 

# 需要导入模块: import singer [as 别名]
# 或者: from singer import parse_message [as 别名]
def test_parse_message_record_with_version_good(self):
        message = singer.parse_message(
            '{"type": "RECORD", "record": {"name": "foo"}, "stream": "users", "version": 2}')
        self.assertEqual(
            message,
            singer.RecordMessage(record={'name': 'foo'}, stream='users', version=2)) 

# 需要导入模块: import singer [as 别名]
# 或者: from singer import parse_message [as 别名]
def test_parse_message_record_naive_extraction_time(self):
        with self.assertRaisesRegex(ValueError, "must be either None or an aware datetime"):
            message = singer.parse_message(
                '{"type": "RECORD", "record": {"name": "foo"}, "stream": "users", "version": 2, "time_extracted": "1970-01-02T00:00:00"}') 

# 需要导入模块: import singer [as 别名]
# 或者: from singer import parse_message [as 别名]
def test_parse_message_record_aware_extraction_time(self):
        message = singer.parse_message(
            '{"type": "RECORD", "record": {"name": "foo"}, "stream": "users", "version": 2, "time_extracted": "1970-01-02T00:00:00.000Z"}')
        expected = singer.RecordMessage(
            record={'name': 'foo'},
            stream='users',
            version=2,
            time_extracted=dateutil.parser.parse("1970-01-02T00:00:00.000Z"))
        print(message)
        print(expected)
        self.assertEqual(message, expected) 

# 需要导入模块: import singer [as 别名]
# 或者: from singer import parse_message [as 别名]
def test_parse_message_record_missing_record(self):
        with self.assertRaises(Exception):
            singer.parse_message('{"type": "RECORD", "stream": "users"}') 

# 需要导入模块: import singer [as 别名]
# 或者: from singer import parse_message [as 别名]
def test_parse_message_schema_good(self):
        message = singer.parse_message('{"type": "SCHEMA", "stream": "users", "schema": {"type": "object", "properties": {"name": {"type": "string"}}}, "key_properties": ["name"]}')  # nopep8
        self.assertEqual(
            message,
            singer.SchemaMessage(
                stream='users',
                key_properties=['name'],
                schema={'type': 'object',
                        'properties': {
                            'name': {'type': 'string'}}})) 

# 需要导入模块: import singer [as 别名]
# 或者: from singer import parse_message [as 别名]
def test_parse_message_schema_missing_stream(self):
        with self.assertRaises(Exception):
            message = singer.parse_message('{"type": "SCHEMA", "schema": {"type": "object", "properties": {"name": {"type": "string"}}}, "key_properties": ["name"]}')  # nopep8 

# 需要导入模块: import singer [as 别名]
# 或者: from singer import parse_message [as 别名]
def test_parse_message_schema_missing_schema(self):
        with self.assertRaises(Exception):
            message = singer.parse_message(
                '{"type": "SCHEMA", "stream": "users", "key_properties": ["name"]}')  # nopep8 

# 需要导入模块: import singer [as 别名]
# 或者: from singer import parse_message [as 别名]
def test_parse_message_schema_missing_key_properties(self):
        with self.assertRaises(Exception):
            message = singer.parse_message('{"type": "SCHEMA", "stream": "users", "schema": {"type": "object", "properties": {"name": {"type": "string"}}}}')  # nopep8 

# 需要导入模块: import singer [as 别名]
# 或者: from singer import parse_message [as 别名]
def test_parse_message_state_good(self):
        message = singer.parse_message(
            '{"type": "STATE", "value": {"seq": 1}}')
        self.assertEqual(message, singer.StateMessage(value={'seq': 1})) 

# 需要导入模块: import singer [as 别名]
# 或者: from singer import parse_message [as 别名]
def create_record(self, value):
        raw = '{"type": "RECORD", "stream": "test", "record": {"value": ' + value + '}}'
        parsed = singer.parse_message(raw)
        return parsed.record['value'] 

# 需要导入模块: import singer [as 别名]
# 或者: from singer import parse_message [as 别名]
def persist_lines(service, spreadsheet, lines):
    state = None
    schemas = {}
    key_properties = {}

    headers_by_stream = {}
    
    for line in lines:
        try:
            msg = singer.parse_message(line)
        except json.decoder.JSONDecodeError:
            logger.error("Unable to parse:\n{}".format(line))
            raise

        if isinstance(msg, singer.RecordMessage):
            if msg.stream not in schemas:
                raise Exception("A record for stream {} was encountered before a corresponding schema".format(msg.stream))

            schema = schemas[msg.stream]
            validate(msg.record, schema)
            flattened_record = flatten(msg.record)
            
            matching_sheet = [s for s in spreadsheet['sheets'] if s['properties']['title'] == msg.stream]
            new_sheet_needed = len(matching_sheet) == 0
            range_name = "{}!A1:ZZZ".format(msg.stream)
            append = functools.partial(append_to_sheet, service, spreadsheet['spreadsheetId'], range_name)

            if new_sheet_needed:
                add_sheet(service, spreadsheet['spreadsheetId'], msg.stream)
                spreadsheet = get_spreadsheet(service, spreadsheet['spreadsheetId']) # refresh this for future iterations
                headers_by_stream[msg.stream] = list(flattened_record.keys())
                append(headers_by_stream[msg.stream])

            elif msg.stream not in headers_by_stream:
                first_row = get_values(service, spreadsheet['spreadsheetId'], range_name + '1')
                if 'values' in first_row:
                    headers_by_stream[msg.stream] = first_row.get('values', None)[0]
                else:
                    headers_by_stream[msg.stream] = list(flattened_record.keys())
                    append(headers_by_stream[msg.stream])

            result = append([flattened_record.get(x, None) for x in headers_by_stream[msg.stream]]) # order by actual headers found in sheet

            state = None
        elif isinstance(msg, singer.StateMessage):
            logger.debug('Setting state to {}'.format(msg.value))
            state = msg.value
        elif isinstance(msg, singer.SchemaMessage):
            schemas[msg.stream] = msg.schema
            key_properties[msg.stream] = msg.key_properties
        else:
            raise Exception("Unrecognized message {}".format(msg))

    return state