Python singer.RecordMessage方法代码示例

# 需要导入模块: import singer [as 别名]
# 或者: from singer import RecordMessage [as 别名]
def add(self, message):
        if isinstance(message, singer.RecordMessage):
            stream = self.ensure_stream(message.stream)
            if stream.latest_schema:
                validator_fn = extend_with_default(Draft4Validator)
                validator = validator_fn(
                    stream.latest_schema, format_checker=FormatChecker())
                validator.validate(copy.deepcopy(message.record))
            else:
                print('I saw a record for stream {} before the schema'.format(
                    message.stream))
                exit(1)
            stream.num_records += 1

        elif isinstance(message, singer.SchemaMessage):
            stream = self.ensure_stream(message.stream)
            stream.num_schemas += 1
            stream.latest_schema = message.schema

        elif isinstance(message, singer.StateMessage):
            self.latest_state = message.value
            self.num_states += 1 

# 需要导入模块: import singer [as 别名]
# 或者: from singer import RecordMessage [as 别名]
def setUp(self):
        self.schema = {
            'type': 'object',
            'properties': {
                'id': {'type': 'integer'},
                'color': {'type': 'string'}
            }
        }

        self.colors = ['red', 'orange', 'yellow', 'green', 'blue', 'indigo', 'violet']
        self.key_names = ['id']
        self.bookmark_names = ['updated_at']

        self.records = [{'id': i, 'color': color, 'updated_at': utils.strftime(utils.now())}
                        for i, color in enumerate(self.colors)]
        self.messages = [RecordMessage(stream='colors', record=r) for r in self.records]
        self.messages.append(ActivateVersionMessage(stream='colors', version=1)) 

# 需要导入模块: import singer [as 别名]
# 或者: from singer import RecordMessage [as 别名]
def test_with_initial_full_table_complete_in_state(self):
        common.get_stream_version = lambda a, b: 12345

        state = {
            'bookmarks': {
                'tap_mysql_test-full_table': {
                    'initial_full_table_complete': True
                }
            }
        }

        global SINGER_MESSAGES
        SINGER_MESSAGES.clear()
        tap_mysql.do_sync(self.conn, {}, self.catalog, state)

        (message_types, versions) = message_types_and_versions(SINGER_MESSAGES)

        self.assertEqual(['RecordMessage', 'ActivateVersionMessage'], message_types)
        self.assertEqual(versions, [12345, 12345]) 

# 需要导入模块: import singer [as 别名]
# 或者: from singer import RecordMessage [as 别名]
def test_version_cleared_from_state_after_full_table_success(self):
        common.get_stream_version = lambda a, b: 12345

        state = {
            'bookmarks': {
                'tap_mysql_test-full_table': {
                    'version': 1,
                    'initial_full_table_complete': True
                }
            }
        }

        global SINGER_MESSAGES
        SINGER_MESSAGES.clear()
        tap_mysql.do_sync(self.conn, {}, self.catalog, state)

        (message_types, versions) = message_types_and_versions(SINGER_MESSAGES)

        self.assertEqual(['RecordMessage', 'ActivateVersionMessage'], message_types)
        self.assertEqual(versions, [12345, 12345])

        self.assertFalse('version' in state['bookmarks']['tap_mysql_test-full_table'].keys())
        self.assertTrue(state['bookmarks']['tap_mysql_test-full_table']['initial_full_table_complete']) 

# 需要导入模块: import singer [as 别名]
# 或者: from singer import RecordMessage [as 别名]
def test_with_no_state(self):
        state = {}

        global SINGER_MESSAGES
        SINGER_MESSAGES.clear()

        tap_mysql.do_sync(self.conn, {}, self.catalog, state)

        (message_types, versions) = message_types_and_versions(SINGER_MESSAGES)

        self.assertEqual(
            ['ActivateVersionMessage',
             'RecordMessage',
             'RecordMessage',
             'RecordMessage',
             'ActivateVersionMessage',
             'RecordMessage',
             'RecordMessage',
             'RecordMessage'],
            message_types)
        self.assertTrue(isinstance(versions[0], int))
        self.assertEqual(versions[0], versions[1]) 

# 需要导入模块: import singer [as 别名]
# 或者: from singer import RecordMessage [as 别名]
def test_round_trip(self):
        record_message = singer.RecordMessage(
            record={'name': 'foo'},
            stream='users')

        schema_message = singer.SchemaMessage(
            stream='users',
            key_properties=['name'],
            schema={'type': 'object',
                    'properties': {
                        'name': {'type': 'string'}}})

        state_message = singer.StateMessage(value={'seq': 1})

        self.assertEqual(record_message,
                         singer.parse_message(singer.format_message(record_message)))
        self.assertEqual(schema_message,
                         singer.parse_message(singer.format_message(schema_message)))
        self.assertEqual(state_message,
                         singer.parse_message(singer.format_message(state_message)))

    ## These three tests just confirm that writing doesn't throw 

# 需要导入模块: import singer [as 别名]
# 或者: from singer import RecordMessage [as 别名]
def handle_batch(self, messages, contains_activate_version, schema, key_names, bookmark_names=None, state_writer=None, state=None): # pylint: disable=no-self-use,unused-argument
        '''Handles messages by validating them against schema.'''
        LOGGER.info("ValidatingHandler handle_batch")
        validator = Draft4Validator(schema, format_checker=FormatChecker())
        for i, message in enumerate(messages):
            if isinstance(message, singer.RecordMessage):
                try:
                    validator.validate(message.record)
                    if key_names:
                        for k in key_names:
                            if k not in message.record:
                                raise TargetStitchException(
                                    'Message {} is missing key property {}'.format(
                                        i, k))
                except Exception as e:
                    raise TargetStitchException(
                        'Record does not pass schema validation: {}'.format(e))

        # pylint: disable=undefined-loop-variable
        # NB: This seems incorrect as there's a chance message is not defined
        LOGGER.info('%s (%s): Batch is valid',
                    messages[0].stream,
                    len(messages))
        if state:
            line = simplejson.dumps(state)
            state_writer.write("{}\n".format(line))
            state_writer.flush() 

# 需要导入模块: import singer [as 别名]
# 或者: from singer import RecordMessage [as 别名]
def overloaded_parse_message(msg):
    """Parse a message string into a Message object."""

    # We are not using Decimals for parsing here.
    # We recognize that exposes data to potentially
    # lossy conversions.  However, this will affect
    # very few data points and we have chosen to
    # leave conversion as is for now.
    obj = simplejson.loads(msg, use_decimal=True)
    msg_type = _required_key(obj, 'type')

    if msg_type == 'RECORD':
        time_extracted = obj.get('time_extracted')
        if time_extracted:
            try:
                time_extracted = ciso8601.parse_datetime(time_extracted)
            except Exception:
                time_extracted = None
        return singer.RecordMessage(stream=_required_key(obj, 'stream'),
                                    record=_required_key(obj, 'record'),
                                    version=obj.get('version'),
                                    time_extracted=time_extracted)

    if msg_type == 'SCHEMA':
        return singer.SchemaMessage(stream=_required_key(obj, 'stream'),
                                    schema=_required_key(obj, 'schema'),
                                    key_properties=_required_key(obj, 'key_properties'),
                                    bookmark_properties=obj.get('bookmark_properties'))

    if msg_type == 'STATE':
        return singer.StateMessage(value=_required_key(obj, 'value'))

    if msg_type == 'ACTIVATE_VERSION':
        return singer.ActivateVersionMessage(stream=_required_key(obj, 'stream'),
                                             version=_required_key(obj, 'version'))
    return None 

# 需要导入模块: import singer [as 别名]
# 或者: from singer import RecordMessage [as 别名]
def test_raises_if_cant_stay_in_limit(self):
        data = 'a' * 21000000
        message = RecordMessage(stream='colors', record=data)
        with self.assertRaisesRegex(target_stitch.BatchTooLargeException, re.compile('the Stitch API limit of 20 Mb')):
            target_stitch.serialize([message], self.schema, self.key_names, self.bookmark_names, 4000000, target_stitch.DEFAULT_MAX_BATCH_RECORDS) 

# 需要导入模块: import singer [as 别名]
# 或者: from singer import RecordMessage [as 别名]
def row_to_singer_record(catalog_entry, version, db_column_map, row, time_extracted):
    row_to_persist = {}

    for column_name, val in row.items():
        property_type = catalog_entry.schema.properties[column_name].type
        db_column_type = db_column_map.get(column_name)

        if isinstance(val, (datetime.datetime, datetime.date, datetime.timedelta)):
            the_utc_date = common.to_utc_datetime_str(val)
            row_to_persist[column_name] = the_utc_date

        elif db_column_type == FIELD_TYPE.JSON:
            row_to_persist[column_name] = json.dumps(json_bytes_to_string(val))

        elif 'boolean' in property_type or property_type == 'boolean':
            if val is None:
                boolean_representation = None
            elif val == 0:
                boolean_representation = False
            elif db_column_type == FIELD_TYPE.BIT:
                boolean_representation = int(val) != 0
            else:
                boolean_representation = True
            row_to_persist[column_name] = boolean_representation

        else:
            row_to_persist[column_name] = val

    return singer.RecordMessage(
        stream=catalog_entry.stream,
        record=row_to_persist,
        version=version,
        time_extracted=time_extracted) 

# 需要导入模块: import singer [as 别名]
# 或者: from singer import RecordMessage [as 别名]
def row_to_singer_record(catalog_entry, version, row, columns, time_extracted):
    row_to_persist = ()
    for idx, elem in enumerate(row):
        property_type = catalog_entry.schema.properties[columns[idx]].type

        if isinstance(elem, (datetime.datetime, datetime.date, datetime.timedelta)):
            the_utc_date = to_utc_datetime_str(elem)
            row_to_persist += (the_utc_date,)

        elif isinstance(elem, bytes):
            # for BIT value, treat 0 as False and anything else as True
            boolean_representation = elem != b'\x00'
            row_to_persist += (boolean_representation,)

        elif 'boolean' in property_type or property_type == 'boolean':
            if elem is None:
                boolean_representation = None
            elif elem == 0:
                boolean_representation = False
            else:
                boolean_representation = True
            row_to_persist += (boolean_representation,)

        else:
            row_to_persist += (elem,)
    rec = dict(zip(columns, row_to_persist))

    return singer.RecordMessage(
        stream=catalog_entry.stream,
        record=rec,
        version=version,
        time_extracted=time_extracted) 

# 需要导入模块: import singer [as 别名]
# 或者: from singer import RecordMessage [as 别名]
def test_initial_full_table(self):
        state = {}
        expected_log_file, expected_log_pos = binlog.fetch_current_log_file_and_pos(self.conn)

        global SINGER_MESSAGES
        SINGER_MESSAGES.clear()
        tap_mysql.do_sync(self.conn, {}, self.catalog, state)

        message_types = [type(m) for m in SINGER_MESSAGES]

        
        self.assertEqual(message_types,
                         [singer.StateMessage,
                          singer.SchemaMessage,
                          singer.ActivateVersionMessage,
                          singer.RecordMessage,
                          singer.RecordMessage,
                          singer.StateMessage,
                          singer.ActivateVersionMessage,
                          singer.StateMessage])

        record_messages = list(filter(lambda m: isinstance(m, singer.RecordMessage), SINGER_MESSAGES))

        # Expected from 0.7.11
        expected_records = [
            {'datetime_col': None,
             'id': 1,
             'timestamp_col': None,
             'time_col': '1970-01-01T00:00:00.000000Z',
             'date_col': None},
            {'datetime_col': None,
             'id': 2,
             'timestamp_col': None,
             'time_col': None,
             'date_col': None}
        ]

        self.assertEqual(expected_records, [x.asdict()['record'] for x in record_messages]) 

# 需要导入模块: import singer [as 别名]
# 或者: from singer import RecordMessage [as 别名]
def message_types_and_versions(messages):
    message_types = []
    versions = []
    for message in messages:
        t = type(message)
        if t in set([singer.RecordMessage, singer.ActivateVersionMessage]):
            message_types.append(t.__name__)
            versions.append(message.version)
    return (message_types, versions) 

# 需要导入模块: import singer [as 别名]
# 或者: from singer import RecordMessage [as 别名]
def test_with_state(self):
        state = {
            'bookmarks': {
                'tap_mysql_test-incremental': {
                    'version': 1,
                    'replication_key_value': '2017-06-20',
                    'replication_key': 'updated'
                },
                'tap_mysql_test-integer_incremental': {
                    'version': 1,
                    'replication_key_value': 3,
                    'replication_key': 'updated'
                }
            }
        }

        global SINGER_MESSAGES
        SINGER_MESSAGES.clear()
        tap_mysql.do_sync(self.conn, {}, self.catalog, state)

        (message_types, versions) = message_types_and_versions(SINGER_MESSAGES)

        self.assertEqual(
            ['ActivateVersionMessage',
             'RecordMessage',
             'RecordMessage',
             'ActivateVersionMessage',
             'RecordMessage'],
            message_types)
        self.assertTrue(isinstance(versions[0], int))
        self.assertEqual(versions[0], versions[1])
        self.assertEqual(versions[1], 1) 

# 需要导入模块: import singer [as 别名]
# 或者: from singer import RecordMessage [as 别名]
def runTest(self):
        global SINGER_MESSAGES
        SINGER_MESSAGES.clear()
        tap_mysql.do_sync(self.conn, {}, self.catalog, {})

        record_message = list(filter(lambda m: isinstance(m, singer.RecordMessage), SINGER_MESSAGES))[0]

        self.assertTrue(isinstance(record_message, singer.RecordMessage))
        self.assertEqual(record_message.record, {'b c': 1})