本文整理汇总了Python中utils.Utils.get_bq方法的典型用法代码示例。如果您正苦于以下问题：Python Utils.get_bq方法的具体用法？Python Utils.get_bq怎么用？Python Utils.get_bq使用的例子？那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类utils.Utils的用法示例。
在下文中一共展示了Utils.get_bq方法的3个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: start
# 需要导入模块: from utils import Utils [as 别名]
# 或者: from utils.Utils import get_bq [as 别名]
def start(schema, logger):
# initialize table mapping for default table
# BUGBUG: initialize based on query to prod
table_mapping = {
config.DATASET_ID + "." + config.TABLE_ID : [config.DATASET_ID, config.TABLE_ID]
}
datasets = Utils.get_bq().datasets().list(projectId=config.PROJECT_ID).execute()
datasets = datasets.get("datasets", None)
for d in datasets:
ref = d.get("datasetReference", None)
bq_tables = Utils.get_bq().tables().list(projectId=ref.get("projectId"), datasetId=ref.get("datasetId")).execute()
if bq_tables['totalItems'] > 0:
for t in bq_tables.get("tables", None):
ref = t.get("tableReference", None)
dataset_id = ref.get("datasetId", None)
table_id = ref.get("tableId", None)
key = Utils.make_tag(dataset_id, table_id)
table_mapping[key] = [dataset_id, table_id]
print("Initialized tables: %s" % table_mapping)
listener = GnipListener(schema, table_mapping, logger=logger)
while True:
stream = None
try:
# clean gnip headers
_headers = GnipListener.HEADERS
headers = {}
for k, v in _headers.items():
headers[k] = v.strip()
#req = urllib2.Request(config.GNIP_STREAM_URL, headers=GnipListener.HEADERS)
req = urllib2.Request(config.GNIP_STREAM_URL, headers=headers)
response = urllib2.urlopen(req, timeout=(1+GnipListener.KEEP_ALIVE))
decompressor = zlib.decompressobj(16+zlib.MAX_WBITS)
remainder = ''
while True:
tmp = decompressor.decompress(response.read(GnipListener.CHUNK_SIZE))
if tmp == '':
return
[records, remainder] = ''.join([remainder, tmp]).rsplit(NEWLINE,1)
listener.on_data(records)
get_stream(listener)
except:
logger.exception("Unexpected error:");
if stream:
stream.disconnect()
time.sleep(SLEEP_TIME)示例2: get
# 需要导入模块: from utils import Utils [as 别名]
# 或者: from utils.Utils import get_bq [as 别名]
def get(self):
table = self.request.get("table", None)
response = rules.get_rules(**GNIP_RULES_PARAMS)
FORMAT = "%Y%m%d%H%M"
end = datetime.now()
# scope by table
if table:
(project, dataset, table) = Utils.parse_bqid(table)
tag = Utils.make_tag(dataset, table)
response = [r for r in response if r['tag'] == tag]
table = Utils.get_bq().tables().get(projectId=project, datasetId=dataset, tableId=table).execute()
end = float(table['creationTime'])
end = Utils.millis_to_date(end)
start = end - timedelta(days=SEARCH_DAYS)
for r in response:
tag = r['tag']
r['hpt'] = json.dumps({
"publisher": "twitter",
"streamType": "track",
"dataFormat": "activity-streams",
"fromDate": start.strftime(FORMAT),
"toDate": end.strftime(FORMAT),
"rules": [{"tag": tag, "value": r['value']}], "title": tag})
self.response.headers['Content-Type'] = 'application/json'
self.response.out.write(json.dumps(response))示例3: get
# 需要导入模块: from utils import Utils [as 别名]
# 或者: from utils.Utils import get_bq [as 别名]
def get(self, id):
(project, dataset, table) = Utils.parse_bqid(id)
response = Utils.get_bq().tables().get(projectId=project, datasetId=dataset, tableId=table).execute()
created = float(response['creationTime'])
response['creationTime'] = Utils.millis_to_str(created)
updated = float(response['lastModifiedTime'])
response['lastModifiedTime'] = Utils.millis_to_str(updated)
self.response.out.write(JINJA.get_template('table_detail.html').render(response))