本文整理汇总了Python中utils.Utils.scrub方法的典型用法代码示例。如果您正苦于以下问题：Python Utils.scrub方法的具体用法？Python Utils.scrub怎么用？Python Utils.scrub使用的例子？那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类utils.Utils的用法示例。
在下文中一共展示了Utils.scrub方法的3个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: on_data
# 需要导入模块: from utils import Utils [as 别名]
# 或者: from utils.Utils import scrub [as 别名]
def on_data(self, data):
# print "Data type: %s" % type(data)
# print "Data: " + data
records_str = data.strip().split(NEWLINE)
# print "Records: %s" % len(records_str)
for r in records_str:
# Twitter returns data in JSON format - we need to decode it first
# temp_str = json.dumps(data)
# print "********************"
# print "Record type: %s" % type(r)
# print "Record: " + r
record = json.loads(r)
if not record.get('delete', None):
record_scrubbed = Utils.scrub(record)
# print "Scrubbed Record"
# print record_scrubbed
Utils.insert_record(self.client, self.dataset_id, self.table_id, record_scrubbed)
if self.logger:
self.logger.info('@%s: %s' % (record['actor']['preferredUsername'], record['body'].encode('ascii', 'ignore')))
self.count = self.count + 1
return True示例2: on_data
# 需要导入模块: from utils import Utils [as 别名]
# 或者: from utils.Utils import scrub [as 别名]
def on_data(self, data):
# get bulk records, but process individually based on tag-based routing
records_str = data.strip().split(NEWLINE)
for r in records_str:
record = json.loads(r)
if not record.get('delete', None):
table = None
tag = self.get_table_tag(record)
print tag
if tag:
table = self.table_mapping.get(tag, None)
if not table:
table = tag.split(".")
created = self.client.create_table(table[0], table[1], self.schema)
if created:
self.table_mapping[tag] = table
self.logger.info('Created BQ table: %s' % tag)
if not table:
table = self.default_table
record_scrubbed = Utils.scrub(record)
Utils.insert_record(self.client, table[0], table[1], record_scrubbed)
if self.logger:
self.logger.info('@%s: %s (%s.%s)' % (record['actor']['preferredUsername'], record['body'].encode('ascii', 'ignore'), table[0], table[1]))
self.count = self.count + 1
return True示例3: on_data
# 需要导入模块: from utils import Utils [as 别名]
# 或者: from utils.Utils import scrub [as 别名]
def on_data(self, data):
# get bulk records, but process individually based on tag-based routing
records_str = data.strip().split(NEWLINE)
for r in records_str:
record = json.loads(r)
if not record.get('delete', None):
tags = self.get_table_tags(record)
if not tags:
tags = [self.default_table]
# process multiple tags on a record
for tag in tags:
table = None
if not tag:
table = self.default_table
else:
table = self.table_mapping.get(tag, None)
if not table:
table = tag.split(".")
created = Utils.insert_table(table[0], table[1], self.schema)
# Brand new table
if created and created != True:
self.logger.info('Created BQ table: %s' % tag)
self.table_mapping[tag] = table
record_scrubbed = Utils.scrub(record)
Utils.insert_records(table[0], table[1], [record_scrubbed])
if self.logger:
self.logger.info('@%s: %s (%s)' % (record['actor']['preferredUsername'], record['body'].encode('ascii', 'ignore'), tags))
self.count = self.count + 1
return True