本文整理汇总了Python中distributed.Executor._gather方法的典型用法代码示例。如果您正苦于以下问题：Python Executor._gather方法的具体用法？Python Executor._gather怎么用？Python Executor._gather使用的例子？那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类distributed.Executor的用法示例。
在下文中一共展示了Executor._gather方法的7个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: dont_test_dataframes
# 需要导入模块: from distributed import Executor [as 别名]
# 或者: from distributed.Executor import _gather [as 别名]
def dont_test_dataframes(s, a): # slow
pytest.importorskip('pandas')
n = 3000000
fn = '/tmp/test/file.csv'
with make_hdfs() as hdfs:
data = (b'name,amount,id\r\n' +
b'Alice,100,1\r\nBob,200,2\r\n' * n)
with hdfs.open(fn, 'w') as f:
f.write(data)
e = Executor((s.ip, s.port), start=False)
yield e._start()
futures = read_bytes(fn, hdfs=hdfs, delimiter=b'\r\n')
assert len(futures) > 1
def load(b, **kwargs):
assert b
from io import BytesIO
import pandas as pd
bio = BytesIO(b)
return pd.read_csv(bio, **kwargs)
dfs = e.map(load, futures, names=['name', 'amount', 'id'], skiprows=1)
dfs2 = yield e._gather(dfs)
assert sum(map(len, dfs2)) == n * 2 - 1示例2: test_lazy_values
# 需要导入模块: from distributed import Executor [as 别名]
# 或者: from distributed.Executor import _gather [as 别名]
def test_lazy_values(s, a, b):
with make_hdfs() as hdfs:
data = b'a'
for i in range(3):
hdfs.mkdir('/tmp/test/data-%d' % i)
for j in range(2):
fn = '/tmp/test/data-%d/file-%d.csv' % (i, j)
with hdfs.open(fn, 'w', repl=1) as f:
f.write(data)
e = Executor((s.ip, s.port), start=False)
yield e._start()
values = read_bytes('/tmp/test/', hdfs=hdfs, lazy=True)
assert all(isinstance(v, Value) for v in values)
while not s.restrictions:
yield gen.sleep(0.01)
assert not s.dask
results = e.compute(*values, sync=False)
results = yield e._gather(results)
assert len(results) == 6
assert all(x == b'a' for x in results)示例3: test_avro
# 需要导入模块: from distributed import Executor [as 别名]
# 或者: from distributed.Executor import _gather [as 别名]
def test_avro(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
avro_files = {'/tmp/test/1.avro': avro_bytes,
'/tmp/test/2.avro': avro_bytes}
with make_hdfs() as hdfs:
for k, v in avro_files.items():
with hdfs.open(k, 'w') as f:
f.write(v)
assert hdfs.info(k)['size'] > 0
L = yield _read_avro('/tmp/test/*.avro', lazy=False)
assert isinstance(L, list)
assert all(isinstance(x, Future) for x in L)
results = yield e._gather(L)
assert all(isinstance(r, list) for r in results)
assert results[0][:5] == data[:5]
assert results[-1][-5:] == data[-5:]
L = yield _read_avro('/tmp/test/*.avro', lazy=True)
assert isinstance(L, list)
assert all(isinstance(x, Value) for x in L)示例4: test_read_bytes
# 需要导入模块: from distributed import Executor [as 别名]
# 或者: from distributed.Executor import _gather [as 别名]
def test_read_bytes(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
futures = read_bytes(test_bucket_name, prefix='test/', anon=True)
assert len(futures) >= len(files)
results = yield e._gather(futures)
assert set(results).issuperset(set(files.values()))
yield e._shutdown()示例5: test_read_bytes_lazy
# 需要导入模块: from distributed import Executor [as 别名]
# 或者: from distributed.Executor import _gather [as 别名]
def test_read_bytes_lazy(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
values = read_bytes(test_bucket_name, 'test/', lazy=True, anon=True)
assert all(isinstance(v, Value) for v in values)
results = e.compute(values, sync=False)
results = yield e._gather(results)
assert set(results).issuperset(set(files.values()))
yield e._shutdown()示例6: test_read_bytes
# 需要导入模块: from distributed import Executor [as 别名]
# 或者: from distributed.Executor import _gather [as 别名]
def test_read_bytes(s, a, b):
with make_hdfs() as hdfs:
data = b'a' * int(1e8)
fn = '/tmp/test/file'
with hdfs.open(fn, 'w', repl=1) as f:
f.write(data)
blocks = hdfs.get_block_locations(fn)
assert len(blocks) > 1
e = Executor((s.ip, s.port), start=False)
yield e._start()
futures = read_bytes(fn, hdfs=hdfs)
assert len(futures) == len(blocks)
assert futures[0].executor is e
results = yield e._gather(futures)
assert b''.join(results) == data
assert s.restrictions
assert {f.key for f in futures}.issubset(s.loose_restrictions)示例7: test_get_block_locations_nested
# 需要导入模块: from distributed import Executor [as 别名]
# 或者: from distributed.Executor import _gather [as 别名]
def test_get_block_locations_nested(s, a, b):
with make_hdfs() as hdfs:
data = b'a'
for i in range(3):
hdfs.mkdir('/tmp/test/data-%d' % i)
for j in range(2):
fn = '/tmp/test/data-%d/file-%d.csv' % (i, j)
with hdfs.open(fn, 'w', repl=1) as f:
f.write(data)
L = get_block_locations(hdfs, '/tmp/test/')
assert len(L) == 6
e = Executor((s.ip, s.port), start=False)
yield e._start()
futures = read_bytes('/tmp/test/', hdfs=hdfs)
results = yield e._gather(futures)
assert len(results) == 6
assert all(x == b'a' for x in results)