本文整理汇总了Python中pysparkling.Context.count方法的典型用法代码示例。如果您正苦于以下问题：Python Context.count方法的具体用法？Python Context.count怎么用？Python Context.count使用的例子？那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类pysparkling.Context的用法示例。
在下文中一共展示了Context.count方法的12个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: test_filter
# 需要导入模块: from pysparkling import Context [as 别名]
# 或者: from pysparkling.Context import count [as 别名]
def test_filter():
my_rdd = Context().parallelize(
[1, 2, 2, 4, 1, 3, 5, 9],
3,
).filter(lambda x: x % 2 == 0)
print(my_rdd.collect())
print(my_rdd.count())
assert my_rdd.count() == 3示例2: test_gs_textFile_loop
# 需要导入模块: from pysparkling import Context [as 别名]
# 或者: from pysparkling.Context import count [as 别名]
def test_gs_textFile_loop():
random.seed()
fn = '{}/pysparkling_test_{:d}.txt'.format(
GS_TEST_PATH, random.random() * 999999.0)
rdd = Context().parallelize('Line {0}'.format(n) for n in range(200))
rdd.saveAsTextFile(fn)
rdd_check = Context().textFile(fn)
assert (
rdd.count() == rdd_check.count() and
all(e1 == e2 for e1, e2 in zip(rdd.collect(), rdd_check.collect()))
)示例3: test_hdfs_textFile_loop
# 需要导入模块: from pysparkling import Context [as 别名]
# 或者: from pysparkling.Context import count [as 别名]
def test_hdfs_textFile_loop():
random.seed()
fn = '{}/pysparkling_test_{:d}.txt'.format(
HDFS_TEST_PATH, random.random() * 999999.0)
print('HDFS test file: {0}'.format(fn))
rdd = Context().parallelize('Hello World {0}'.format(x) for x in range(10))
rdd.saveAsTextFile(fn)
read_rdd = Context().textFile(fn)
print(rdd.collect())
print(read_rdd.collect())
assert (
rdd.count() == read_rdd.count() and
all(r1 == r2 for r1, r2 in zip(rdd.collect(), read_rdd.collect()))
)示例4: test_hdfs_textFile_loop
# 需要导入模块: from pysparkling import Context [as 别名]
# 或者: from pysparkling.Context import count [as 别名]
def test_hdfs_textFile_loop():
if not HDFS_TEST_PATH:
raise SkipTest
random.seed()
fn = HDFS_TEST_PATH+'/pysparkling_test_{0}.txt'.format(
int(random.random()*999999.0)
)
rdd = Context().parallelize('Hello World {0}'.format(x) for x in range(10))
rdd.saveAsTextFile(fn)
read_rdd = Context().textFile(fn)
assert (
rdd.count() == read_rdd.count() and
all(r1 == r2 for r1, r2 in zip(rdd.collect(), read_rdd.collect()))
)示例5: test_s3_textFile_loop
# 需要导入模块: from pysparkling import Context [as 别名]
# 或者: from pysparkling.Context import count [as 别名]
def test_s3_textFile_loop():
if not AWS_ACCESS_KEY_ID or not S3_TEST_PATH:
raise SkipTest
random.seed()
fn = S3_TEST_PATH+'/pysparkling_test_{0}.txt'.format(
int(random.random()*999999.0)
)
rdd = Context().parallelize("Line {0}".format(n) for n in range(200))
rdd.saveAsTextFile(fn)
rdd_check = Context().textFile(fn)
assert (
rdd.count() == rdd_check.count() and
all(e1 == e2 for e1, e2 in zip(rdd.collect(), rdd_check.collect()))
)示例6: test_gs_textFile_loop
# 需要导入模块: from pysparkling import Context [as 别名]
# 或者: from pysparkling.Context import count [as 别名]
def test_gs_textFile_loop():
if not OAUTH2_CLIENT_ID or not GS_TEST_PATH:
raise SkipTest
random.seed()
fn = '{}/pysparkling_test_{0}.txt'.format(
GS_TEST_PATH, int(random.random() * 999999.0)
)
rdd = Context().parallelize("Line {0}".format(n) for n in range(200))
rdd.saveAsTextFile(fn)
rdd_check = Context().textFile(fn)
assert (
rdd.count() == rdd_check.count() and
all(e1 == e2 for e1, e2 in zip(rdd.collect(), rdd_check.collect()))
)示例7: Context
# 需要导入模块: from pysparkling import Context [as 别名]
# 或者: from pysparkling.Context import count [as 别名]
from pysparkling import Context
my_rdd = Context().textFile("tests/*.py")
print(
"In tests/*.py: all lines={0}, with import={1}".format(
my_rdd.count(), my_rdd.filter(lambda l: l.startswith("import ")).count()
)
)
示例8: test_distinct
# 需要导入模块: from pysparkling import Context [as 别名]
# 或者: from pysparkling.Context import count [as 别名]
def test_distinct():
my_rdd = Context().parallelize([1, 2, 2, 4, 1]).distinct()
assert my_rdd.count() == 3示例9: test_count_partitions
# 需要导入模块: from pysparkling import Context [as 别名]
# 或者: from pysparkling.Context import count [as 别名]
def test_count_partitions():
my_rdd = Context().parallelize([1, 2, 3], 2)
print(my_rdd.collect())
my_rdd.foreach(print)
assert my_rdd.count() == 3示例10: test_count
# 需要导入模块: from pysparkling import Context [as 别名]
# 或者: from pysparkling.Context import count [as 别名]
def test_count():
my_rdd = Context().parallelize([1, 2, 3])
assert my_rdd.count() == 3示例11: Context
# 需要导入模块: from pysparkling import Context [as 别名]
# 或者: from pysparkling.Context import count [as 别名]
from __future__ import print_function
from pysparkling import Context
my_rdd = Context().textFile('tests/*.py')
print('In tests/*.py: all lines={0}, with import={1}'.format(
my_rdd.count(),
my_rdd.filter(lambda l: l.startswith('import ')).count(),
))
示例12: test_parallelize_empty_partitions_at_end
# 需要导入模块: from pysparkling import Context [as 别名]
# 或者: from pysparkling.Context import count [as 别名]
def test_parallelize_empty_partitions_at_end():
my_rdd = Context().parallelize(range(3529), 500)
print(my_rdd.getNumPartitions())
my_rdd.foreachPartition(lambda p: print(sum(1 for _ in p)))
assert my_rdd.getNumPartitions() == 500 and my_rdd.count() == 3529