本文整理匯總了Python中singer.metadata.new方法的典型用法代碼示例。如果您正苦於以下問題：Python metadata.new方法的具體用法？Python metadata.new怎麽用？Python metadata.new使用的例子？那麽, 這裏精選的方法代碼示例或許可以為您提供幫助。您也可以進一步了解該方法所在類singer.metadata的用法示例。
在下文中一共展示了metadata.new方法的6個代碼示例，這些例子默認根據受歡迎程度排序。您可以為喜歡或者感覺有用的代碼點讚，您的評價將有助於係統推薦出更棒的Python代碼示例。
示例1: load_discovered_schema
# 需要導入模塊: from singer import metadata [as 別名]
# 或者: from singer.metadata import new [as 別名]
def load_discovered_schema(stream):
schema = load_schema(stream.tap_stream_id)
mdata = metadata.new()
mdata = metadata.write(mdata, (), 'table-key-properties', stream.key_properties)
mdata = metadata.write(mdata, (), 'forced-replication-method', stream.replication_method)
if stream.replication_key:
mdata = metadata.write(mdata, (), 'valid-replication-keys', [stream.replication_key])
for field_name, props in schema['properties'].items():
if field_name in stream.key_properties or field_name == stream.replication_key:
mdata = metadata.write(mdata, ('properties', field_name), 'inclusion', 'automatic')
else:
mdata = metadata.write(mdata, ('properties', field_name), 'inclusion', 'available')
# The engagements stream has nested data that we synthesize; The engagement field needs to be automatic
if stream.tap_stream_id == "engagements":
mdata = metadata.write(mdata, ('properties', 'engagement'), 'inclusion', 'automatic')
return schema, metadata.to_list(mdata) 示例2: generate_metadata
# 需要導入模塊: from singer import metadata [as 別名]
# 或者: from singer.metadata import new [as 別名]
def generate_metadata(stream, schema):
mdata = metadata.new()
mdata = metadata.write(mdata, (), 'table-key-properties', stream.pk_fields)
for field_name in schema.properties.keys():
if field_name in stream.pk_fields:
mdata = metadata.write(mdata, ('properties', field_name), 'inclusion', 'automatic')
else:
mdata = metadata.write(mdata, ('properties', field_name), 'inclusion', 'available')
return metadata.to_list(mdata) 示例3: load_metadata
# 需要導入模塊: from singer import metadata [as 別名]
# 或者: from singer.metadata import new [as 別名]
def load_metadata(table_spec, schema):
mdata = metadata.new()
mdata = metadata.write(mdata, (), 'table-key-properties', table_spec['key_properties'])
for field_name in schema.get('properties', {}).keys():
if table_spec.get('key_properties', []) and field_name in table_spec.get('key_properties', []):
mdata = metadata.write(mdata, ('properties', field_name), 'inclusion', 'automatic')
else:
mdata = metadata.write(mdata, ('properties', field_name), 'inclusion', 'available')
return metadata.to_list(mdata) 示例4: translate_state
# 需要導入模塊: from singer import metadata [as 別名]
# 或者: from singer.metadata import new [as 別名]
def translate_state(state, catalog, repositories):
'''
This tap used to only support a single repository, in which case the
state took the shape of:
{
"bookmarks": {
"commits": {
"since": "2018-11-14T13:21:20.700360Z"
}
}
}
The tap now supports multiple repos, so this function should be called
at the beginning of each run to ensure the state is translate to the
new format:
{
"bookmarks": {
"singer-io/tap-adwords": {
"commits": {
"since": "2018-11-14T13:21:20.700360Z"
}
}
"singer-io/tap-salesforce": {
"commits": {
"since": "2018-11-14T13:21:20.700360Z"
}
}
}
}
'''
nested_dict = lambda: collections.defaultdict(nested_dict)
new_state = nested_dict()
for stream in catalog['streams']:
stream_name = stream['tap_stream_id']
for repo in repositories:
if bookmarks.get_bookmark(state, repo, stream_name):
return state
if bookmarks.get_bookmark(state, stream_name, 'since'):
new_state['bookmarks'][repo][stream_name]['since'] = bookmarks.get_bookmark(state, stream_name, 'since')
return new_state 示例5: populate_metadata
# 需要導入模塊: from singer import metadata [as 別名]
# 或者: from singer.metadata import new [as 別名]
def populate_metadata(schema_name, schema):
mdata = metadata.new()
#mdata = metadata.write(mdata, (), 'forced-replication-method', KEY_PROPERTIES[schema_name])
mdata = metadata.write(mdata, (), 'table-key-properties', KEY_PROPERTIES[schema_name])
for field_name in schema['properties'].keys():
if field_name in KEY_PROPERTIES[schema_name]:
mdata = metadata.write(mdata, ('properties', field_name), 'inclusion', 'automatic')
else:
mdata = metadata.write(mdata, ('properties', field_name), 'inclusion', 'available')
return mdata 示例6: get_discovery_metadata
# 需要導入模塊: from singer import metadata [as 別名]
# 或者: from singer.metadata import new [as 別名]
def get_discovery_metadata(stream, schema):
mdata = metadata.new()
mdata = metadata.write(mdata, (), 'table-key-properties', stream.key_properties)
mdata = metadata.write(mdata, (), 'forced-replication-method', stream.replication_method)
if stream.replication_key:
mdata = metadata.write(mdata, (), 'valid-replication-keys', [stream.replication_key])
for field_name in schema['properties'].keys():
if field_name in stream.key_properties or field_name == stream.replication_key:
mdata = metadata.write(mdata, ('properties', field_name), 'inclusion', 'automatic')
else:
mdata = metadata.write(mdata, ('properties', field_name), 'inclusion', 'available')
return metadata.to_list(mdata)