本文整理汇总了Python中pyspark.rdd.RDD.mapPartitions方法的典型用法代码示例。如果您正苦于以下问题：Python RDD.mapPartitions方法的具体用法？Python RDD.mapPartitions怎么用？Python RDD.mapPartitions使用的例子？那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类pyspark.rdd.RDD的用法示例。
在下文中一共展示了RDD.mapPartitions方法的3个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: __init__
# 需要导入模块: from pyspark.rdd import RDD [as 别名]
# 或者: from pyspark.rdd.RDD import mapPartitions [as 别名]
def __init__(self, ctx, resource_read=None, query=None, **kwargs):
kwargs = make_es_config(kwargs, resource_read=resource_read, query=query)
kwargs = as_java_object(ctx._gateway, kwargs)
jrdd = helper(ctx).esJsonRDD(ctx._jsc, kwargs)
rdd = RDD(jrdd, ctx, NoOpSerializer())
# read the rdd in batches of two (first key then value / doc)
def pairwise(iterable):
iterator = iter(iterable)
return izip(iterator, iterator)
kvRdd = rdd.mapPartitions(pairwise, True)
super(EsRDD, self).__init__(kvRdd._jrdd, ctx)示例2: rdd
# 需要导入模块: from pyspark.rdd import RDD [as 别名]
# 或者: from pyspark.rdd.RDD import mapPartitions [as 别名]
def rdd(self):
"""Returns the content as an :class:`pyspark.RDD` of :class:`Row`.
"""
if self._lazy_rdd is None:
jrdd = self._jdf.javaToPython()
rdd = RDD(jrdd, self.sql_ctx._sc, BatchedSerializer(PickleSerializer()))
schema = self.schema
def applySchema(it):
cls = _create_cls(schema)
return map(cls, it)
self._lazy_rdd = rdd.mapPartitions(applySchema)
return self._lazy_rdd示例3: rdd
# 需要导入模块: from pyspark.rdd import RDD [as 别名]
# 或者: from pyspark.rdd.RDD import mapPartitions [as 别名]
def rdd(self):
"""
Return the content of the :class:`DataFrame` as an :class:`RDD`
of :class:`Row` s.
"""
if not hasattr(self, '_lazy_rdd'):
jrdd = self._jdf.javaToPython()
rdd = RDD(jrdd, self.sql_ctx._sc, BatchedSerializer(PickleSerializer()))
schema = self.schema
def applySchema(it):
cls = _create_cls(schema)
return itertools.imap(cls, it)
self._lazy_rdd = rdd.mapPartitions(applySchema)
return self._lazy_rdd