本文整理汇总了Python中singer.metadata.to_list方法的典型用法代码示例。如果您正苦于以下问题：Python metadata.to_list方法的具体用法？Python metadata.to_list怎么用？Python metadata.to_list使用的例子？那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类singer.metadata的用法示例。
在下文中一共展示了metadata.to_list方法的8个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: load_discovered_schema
# 需要导入模块: from singer import metadata [as 别名]
# 或者: from singer.metadata import to_list [as 别名]
def load_discovered_schema(stream):
schema = load_schema(stream.tap_stream_id)
mdata = metadata.new()
mdata = metadata.write(mdata, (), 'table-key-properties', stream.key_properties)
mdata = metadata.write(mdata, (), 'forced-replication-method', stream.replication_method)
if stream.replication_key:
mdata = metadata.write(mdata, (), 'valid-replication-keys', [stream.replication_key])
for field_name, props in schema['properties'].items():
if field_name in stream.key_properties or field_name == stream.replication_key:
mdata = metadata.write(mdata, ('properties', field_name), 'inclusion', 'automatic')
else:
mdata = metadata.write(mdata, ('properties', field_name), 'inclusion', 'available')
# The engagements stream has nested data that we synthesize; The engagement field needs to be automatic
if stream.tap_stream_id == "engagements":
mdata = metadata.write(mdata, ('properties', 'engagement'), 'inclusion', 'automatic')
return schema, metadata.to_list(mdata) 示例2: discover_schemas
# 需要导入模块: from singer import metadata [as 别名]
# 或者: from singer.metadata import to_list [as 别名]
def discover_schemas():
# Load Facebook's shared schemas
refs = load_shared_schema_refs()
result = {'streams': []}
streams = initialize_streams_for_discovery()
for stream in streams:
LOGGER.info('Loading schema for %s', stream.name)
schema = singer.resolve_schema_references(load_schema(stream), refs)
mdata = metadata.to_map(metadata.get_standard_metadata(schema,
key_properties=stream.key_properties))
bookmark_key = BOOKMARK_KEYS.get(stream.name)
if bookmark_key == UPDATED_TIME_KEY:
mdata = metadata.write(mdata, ('properties', bookmark_key), 'inclusion', 'automatic')
result['streams'].append({'stream': stream.name,
'tap_stream_id': stream.name,
'schema': schema,
'metadata': metadata.to_list(mdata)})
return result 示例3: do_discover_reports
# 需要导入模块: from singer import metadata [as 别名]
# 或者: from singer.metadata import to_list [as 别名]
def do_discover_reports(sdk_client):
url = 'https://adwords.google.com/api/adwords/reportdownload/{}/reportDefinition.xsd'.format(VERSION)
xsd = request_xsd(url)
root = ET.fromstring(xsd)
nodes = list(root.find(".//*[@name='ReportDefinition.ReportType']/*"))
stream_names = [p.attrib['value'] for p in nodes if p.attrib['value'] in VERIFIED_REPORTS]
streams = []
LOGGER.info("Starting report discovery")
for stream_name in stream_names:
schema, mdata = create_schema_for_report(stream_name, sdk_client)
streams.append({'stream': stream_name,
'tap_stream_id': stream_name,
'metadata' : metadata.to_list(mdata),
'schema': schema})
LOGGER.info("Report discovery complete")
return streams 示例4: get_catalog
# 需要导入模块: from singer import metadata [as 别名]
# 或者: from singer.metadata import to_list [as 别名]
def get_catalog():
raw_schemas = load_schemas()
streams = []
for schema_name, schema in raw_schemas.items():
# get metadata for each field
mdata = populate_metadata(schema_name, schema)
# create and add catalog entry
catalog_entry = {
'stream': schema_name,
'tap_stream_id': schema_name,
'schema': schema,
'metadata' : metadata.to_list(mdata),
'key_properties': KEY_PROPERTIES[schema_name],
}
streams.append(catalog_entry)
return {'streams': streams} 示例5: generate_metadata
# 需要导入模块: from singer import metadata [as 别名]
# 或者: from singer.metadata import to_list [as 别名]
def generate_metadata(stream, schema):
mdata = metadata.new()
mdata = metadata.write(mdata, (), 'table-key-properties', stream.pk_fields)
for field_name in schema.properties.keys():
if field_name in stream.pk_fields:
mdata = metadata.write(mdata, ('properties', field_name), 'inclusion', 'automatic')
else:
mdata = metadata.write(mdata, ('properties', field_name), 'inclusion', 'available')
return metadata.to_list(mdata) 示例6: create_column_metadata
# 需要导入模块: from singer import metadata [as 别名]
# 或者: from singer.metadata import to_list [as 别名]
def create_column_metadata(cols):
mdata = {}
mdata = metadata.write(mdata, (), 'selected-by-default', False)
for c in cols:
schema = schema_for_column(c)
mdata = metadata.write(mdata,
('properties', c.column_name),
'selected-by-default',
schema.inclusion != 'unsupported')
mdata = metadata.write(mdata,
('properties', c.column_name),
'sql-datatype',
c.column_type.lower())
return metadata.to_list(mdata) 示例7: load_metadata
# 需要导入模块: from singer import metadata [as 别名]
# 或者: from singer.metadata import to_list [as 别名]
def load_metadata(table_spec, schema):
mdata = metadata.new()
mdata = metadata.write(mdata, (), 'table-key-properties', table_spec['key_properties'])
for field_name in schema.get('properties', {}).keys():
if table_spec.get('key_properties', []) and field_name in table_spec.get('key_properties', []):
mdata = metadata.write(mdata, ('properties', field_name), 'inclusion', 'automatic')
else:
mdata = metadata.write(mdata, ('properties', field_name), 'inclusion', 'available')
return metadata.to_list(mdata) 示例8: get_discovery_metadata
# 需要导入模块: from singer import metadata [as 别名]
# 或者: from singer.metadata import to_list [as 别名]
def get_discovery_metadata(stream, schema):
mdata = metadata.new()
mdata = metadata.write(mdata, (), 'table-key-properties', stream.key_properties)
mdata = metadata.write(mdata, (), 'forced-replication-method', stream.replication_method)
if stream.replication_key:
mdata = metadata.write(mdata, (), 'valid-replication-keys', [stream.replication_key])
for field_name in schema['properties'].keys():
if field_name in stream.key_properties or field_name == stream.replication_key:
mdata = metadata.write(mdata, ('properties', field_name), 'inclusion', 'automatic')
else:
mdata = metadata.write(mdata, ('properties', field_name), 'inclusion', 'available')
return metadata.to_list(mdata)