本文整理匯總了Python中pyspark.SQLContext.jsonRDD方法的典型用法代碼示例。如果您正苦於以下問題：Python SQLContext.jsonRDD方法的具體用法？Python SQLContext.jsonRDD怎麽用？Python SQLContext.jsonRDD使用的例子？那麽, 這裏精選的方法代碼示例或許可以為您提供幫助。您也可以進一步了解該方法所在類pyspark.SQLContext的用法示例。
在下文中一共展示了SQLContext.jsonRDD方法的2個代碼示例，這些例子默認根據受歡迎程度排序。您可以為喜歡或者感覺有用的代碼點讚，您的評價將有助於係統推薦出更棒的Python代碼示例。
示例1: SparkContext
# 需要導入模塊: from pyspark import SQLContext [as 別名]
# 或者: from pyspark.SQLContext import jsonRDD [as 別名]
sc = SparkContext(conf=conf)
sqlContext = SQLContext(sc)
# 查詢從當前時間開始前10s的數據
now_datetime = datetime.now()
end_time = int(time.mktime(now_datetime.timetuple()))
start_time = int(time.mktime((now_datetime - timedelta(seconds=5)).timetuple()))
print start_time
print end_time
his_data = RedisCache().zrange_by_score("his_data_zadd", start_time, end_time)
#print his_data['result']
if his_data['result']:
hisRDD = sc.parallelize(his_data['result'])
his = sqlContext.jsonRDD(hisRDD)
print his.count()
row = his.sort(his.value.desc()).first()
print "|Time: " + row[0], "|Name: " + row[1], "|Oid: " + row[2], "|Value: " + str(row[3]) + "|"
his.registerTempTable("his_data_zadd")
#sqlContext.cacheTable("his_data_zadd")
#assets = sqlContext.sql("SELECT his.name, his.oid FROM his_data_zadd as his WHERE his.value > 200 AND his.oid < 3000000")
#sql_str = "SELECT his.name, his.oid FROM (SELECT MAX(temp_t.value), temp_t.name, temp_t.oid FROM his_data_zadd AS temp_t) his"
#sql_str = "SELECT his.name, his.oid, his.value FROM his_data_zadd AS his ORDER BY his.value DESC LIMIT 10"
#sql_str = 'SELECT his.name, his.oid FROM his_data_zadd AS his WHERE EXISTS (SELECT MAX(temp_t.value) FROM his_data_zadd AS temp_t)'
"""
Spark 1.5 does not support subquery.示例2: print_fun
# 需要導入模塊: from pyspark import SQLContext [as 別名]
# 或者: from pyspark.SQLContext import jsonRDD [as 別名]
from pyspark import SparkContext, SparkConf
from pyspark import SQLContext
import itertools
def print_fun(collect):
for item in itertools.chain(collect):
print "|Name: " + item[0], "|Value: " + str(item[1]), "|Attribute: " + item[2]
if __name__ == "__main__":
conf = SparkConf().setAppName("json_ops").setMaster("local[3]")
sc = SparkContext(conf=conf)
sqlContext = SQLContext(sc)
# 將json格式直接直接parallelize為RDD
equipmentRDD = sc.parallelize(['{"name":"asset1","value":1,"attribute":"屬性1"}',
'{"name":"asset2","value":2,"attribute":"屬性2"}',
'{"name":"asset3","value":3,"attribute":"屬性3"}'])
equipment = sqlContext.jsonRDD(equipmentRDD)
equipment.registerTempTable("equipment")
assets = sqlContext.sql("SELECT * FROM equipment as eq WHERE eq.value >= 1 AND eq.value <= 2")
assets.show()
# 查詢結果進行隱射
assetMap = assets.map(lambda asset: (asset.name, asset.value, asset.attribute)).foreachPartition(print_fun)
sc.stop()