本文整理汇总了Python中pyspark.streaming.context.StreamingContext.socketTextStream方法的典型用法代码示例。如果您正苦于以下问题：Python StreamingContext.socketTextStream方法的具体用法？Python StreamingContext.socketTextStream怎么用？Python StreamingContext.socketTextStream使用的例子？那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类pyspark.streaming.context.StreamingContext的用法示例。
在下文中一共展示了StreamingContext.socketTextStream方法的3个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: createSSC
# 需要导入模块: from pyspark.streaming.context import StreamingContext [as 别名]
# 或者: from pyspark.streaming.context.StreamingContext import socketTextStream [as 别名]
def createSSC():
# ssc 생성
conf = SparkConf()
sc = SparkContext(master="local[*]", appName="CheckpointSample", conf=conf)
ssc = StreamingContext(sc, 3)
# DStream 생성
ids1 = ssc.socketTextStream("127.0.0.1", 9000)
ids2 = ids1.flatMap(lambda v: v.split(" ")).map(lambda v: (v, 1))
# updateStateByKey
ids2.updateStateByKey(updateFunc).pprint()
# checkpoint
ssc.checkpoint("./checkPoints/checkPointSample/Python")
# return
return ssc示例2: SparkConf
# 需要导入模块: from pyspark.streaming.context import StreamingContext [as 别名]
# 或者: from pyspark.streaming.context.StreamingContext import socketTextStream [as 别名]
# -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, division, unicode_literals
import sys
from pyspark import SparkConf, SparkContext
from pyspark.storagelevel import StorageLevel
from pyspark.streaming.context import StreamingContext
if __name__ == '__main__':
conf = SparkConf().setAppName('Network Word Count')
sc = SparkContext(conf=conf)
ssc = StreamingContext(sc, 5)
lines = ssc.socketTextStream(sys.argv[1], int(sys.argv[2]), StorageLevel.MEMORY_AND_DISK_SER)
word_counts = (
lines.flatMap(lambda l: l.split(' '))
.map(lambda w: (w, 1L))
.reduceByKey(lambda x, y: x + y)
)
word_counts.pprint()
ssc.start()
ssc.awaitTermination()
示例3: SparkConf
# 需要导入模块: from pyspark.streaming.context import StreamingContext [as 别名]
# 或者: from pyspark.streaming.context.StreamingContext import socketTextStream [as 别名]
# -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, division, unicode_literals
import sys
from pyspark import SparkConf, SparkContext
from pyspark.streaming.context import StreamingContext
if __name__ == '__main__':
conf = SparkConf().setAppName('Stateful Network Word Count')
sc = SparkContext(conf=conf)
ssc = StreamingContext(sc, 5)
ssc.checkpoint("checkpoint")
def update(values, state):
return (state or 0) + sum(values)
lines = ssc.socketTextStream(sys.argv[1], int(sys.argv[2]))
word_counts = (
lines.flatMap(lambda l: l.split(' '))
.map(lambda w: (w, 1L))
.updateStateByKey(update)
)
word_counts.pprint()
ssc.start()
ssc.awaitTermination()