本文整理汇总了Python中singer.metadata.new方法的典型用法代码示例。如果您正苦于以下问题：Python metadata.new方法的具体用法？Python metadata.new怎么用？Python metadata.new使用的例子？那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类singer.metadata的用法示例。
在下文中一共展示了metadata.new方法的6个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: load_discovered_schema
# 需要导入模块: from singer import metadata [as 别名]
# 或者: from singer.metadata import new [as 别名]
def load_discovered_schema(stream):
schema = load_schema(stream.tap_stream_id)
mdata = metadata.new()
mdata = metadata.write(mdata, (), 'table-key-properties', stream.key_properties)
mdata = metadata.write(mdata, (), 'forced-replication-method', stream.replication_method)
if stream.replication_key:
mdata = metadata.write(mdata, (), 'valid-replication-keys', [stream.replication_key])
for field_name, props in schema['properties'].items():
if field_name in stream.key_properties or field_name == stream.replication_key:
mdata = metadata.write(mdata, ('properties', field_name), 'inclusion', 'automatic')
else:
mdata = metadata.write(mdata, ('properties', field_name), 'inclusion', 'available')
# The engagements stream has nested data that we synthesize; The engagement field needs to be automatic
if stream.tap_stream_id == "engagements":
mdata = metadata.write(mdata, ('properties', 'engagement'), 'inclusion', 'automatic')
return schema, metadata.to_list(mdata) 示例2: generate_metadata
# 需要导入模块: from singer import metadata [as 别名]
# 或者: from singer.metadata import new [as 别名]
def generate_metadata(stream, schema):
mdata = metadata.new()
mdata = metadata.write(mdata, (), 'table-key-properties', stream.pk_fields)
for field_name in schema.properties.keys():
if field_name in stream.pk_fields:
mdata = metadata.write(mdata, ('properties', field_name), 'inclusion', 'automatic')
else:
mdata = metadata.write(mdata, ('properties', field_name), 'inclusion', 'available')
return metadata.to_list(mdata) 示例3: load_metadata
# 需要导入模块: from singer import metadata [as 别名]
# 或者: from singer.metadata import new [as 别名]
def load_metadata(table_spec, schema):
mdata = metadata.new()
mdata = metadata.write(mdata, (), 'table-key-properties', table_spec['key_properties'])
for field_name in schema.get('properties', {}).keys():
if table_spec.get('key_properties', []) and field_name in table_spec.get('key_properties', []):
mdata = metadata.write(mdata, ('properties', field_name), 'inclusion', 'automatic')
else:
mdata = metadata.write(mdata, ('properties', field_name), 'inclusion', 'available')
return metadata.to_list(mdata) 示例4: translate_state
# 需要导入模块: from singer import metadata [as 别名]
# 或者: from singer.metadata import new [as 别名]
def translate_state(state, catalog, repositories):
'''
This tap used to only support a single repository, in which case the
state took the shape of:
{
"bookmarks": {
"commits": {
"since": "2018-11-14T13:21:20.700360Z"
}
}
}
The tap now supports multiple repos, so this function should be called
at the beginning of each run to ensure the state is translate to the
new format:
{
"bookmarks": {
"singer-io/tap-adwords": {
"commits": {
"since": "2018-11-14T13:21:20.700360Z"
}
}
"singer-io/tap-salesforce": {
"commits": {
"since": "2018-11-14T13:21:20.700360Z"
}
}
}
}
'''
nested_dict = lambda: collections.defaultdict(nested_dict)
new_state = nested_dict()
for stream in catalog['streams']:
stream_name = stream['tap_stream_id']
for repo in repositories:
if bookmarks.get_bookmark(state, repo, stream_name):
return state
if bookmarks.get_bookmark(state, stream_name, 'since'):
new_state['bookmarks'][repo][stream_name]['since'] = bookmarks.get_bookmark(state, stream_name, 'since')
return new_state 示例5: populate_metadata
# 需要导入模块: from singer import metadata [as 别名]
# 或者: from singer.metadata import new [as 别名]
def populate_metadata(schema_name, schema):
mdata = metadata.new()
#mdata = metadata.write(mdata, (), 'forced-replication-method', KEY_PROPERTIES[schema_name])
mdata = metadata.write(mdata, (), 'table-key-properties', KEY_PROPERTIES[schema_name])
for field_name in schema['properties'].keys():
if field_name in KEY_PROPERTIES[schema_name]:
mdata = metadata.write(mdata, ('properties', field_name), 'inclusion', 'automatic')
else:
mdata = metadata.write(mdata, ('properties', field_name), 'inclusion', 'available')
return mdata 示例6: get_discovery_metadata
# 需要导入模块: from singer import metadata [as 别名]
# 或者: from singer.metadata import new [as 别名]
def get_discovery_metadata(stream, schema):
mdata = metadata.new()
mdata = metadata.write(mdata, (), 'table-key-properties', stream.key_properties)
mdata = metadata.write(mdata, (), 'forced-replication-method', stream.replication_method)
if stream.replication_key:
mdata = metadata.write(mdata, (), 'valid-replication-keys', [stream.replication_key])
for field_name in schema['properties'].keys():
if field_name in stream.key_properties or field_name == stream.replication_key:
mdata = metadata.write(mdata, ('properties', field_name), 'inclusion', 'automatic')
else:
mdata = metadata.write(mdata, ('properties', field_name), 'inclusion', 'available')
return metadata.to_list(mdata)