Python hdfs3.HDFileSystem类代码示例

def read_block_from_hdfs(filename, offset, length, host=None, port=None,
        delimiter=None):
    from locket import lock_file
    with lock_file('.lock'):
        hdfs = HDFileSystem(host=host, port=port)
        bytes = hdfs.read_block(filename, offset, length, delimiter)
    return bytes

def test_connection_error():
    with pytest.raises(RuntimeError) as ctx:
        hdfs = HDFileSystem(host='localhost', port=9999, connect=False)
        hdfs.connect()
    # error message is long and with java exceptions, so here we just check
    # that important part of error is present
    msg = 'Caused by: HdfsNetworkConnectException: Connect to "localhost:9999"'
    assert msg in str(ctx.value)

def read_block_from_hdfs(host, port, filename, offset, length, delimiter):
    from hdfs3 import HDFileSystem
    if sys.version_info[0] == 2:
        from locket import lock_file
        with lock_file('.lock'):
            hdfs = HDFileSystem(host=host, port=port)
            bytes = hdfs.read_block(filename, offset, length, delimiter)
    else:
        hdfs = HDFileSystem(host=host, port=port)
        bytes = hdfs.read_block(filename, offset, length, delimiter)
    return bytes

def hdfs():
    hdfs = HDFileSystem(host='localhost', port=8020)
    if hdfs.exists('/tmp/test'):
        hdfs.rm('/tmp/test')
    hdfs.mkdir('/tmp/test')

    yield hdfs

    if hdfs.exists('/tmp/test'):
        hdfs.rm('/tmp/test')

def make_hdfs():
    hdfs = HDFileSystem(host='localhost', port=8020)
    if hdfs.exists('/tmp/test'):
        hdfs.rm('/tmp/test')
    hdfs.mkdir('/tmp/test')

    try:
        yield hdfs
    finally:
        if hdfs.exists('/tmp/test'):
            hdfs.rm('/tmp/test')

  def __init__(self):
    """Initializes a connection to HDFS.

    Connection configuration is done using :doc:`hdfs`.
    """
    super(HadoopFileSystem, self).__init__()
    self._hdfs_client = HDFileSystem()

def make_hdfs():
    from hdfs3 import HDFileSystem
    # from .hdfs import DaskHDFileSystem
    basedir = '/tmp/test-distributed'
    hdfs = HDFileSystem(host='localhost', port=8020)
    if hdfs.exists(basedir):
        hdfs.rm(basedir)
    hdfs.mkdir(basedir)

    try:
        yield hdfs, basedir
    finally:
        if hdfs.exists(basedir):
            hdfs.rm(basedir)

 def get_block_locations(self, paths):
     offsets = []
     lengths = []
     machines = []
     for path in paths:
         if path.startswith('hdfs://'):
             path = path[len('hdfs://'):]
         out = HDFileSystem.get_block_locations(self, path)
         offsets.append([o['offset'] for o in out])
         lengths.append([o['length'] for o in out])
         machines.append([o['hosts'] for o in out])
     return offsets, lengths, machines

def hdfs_open_file(path, auth):
    hdfs = HDFileSystem(**auth)
    return hdfs.open(path, mode='rb')

from hdfs3 import HDFileSystem
import os

os.environp['HADOOP_USER_NAME']=hadoop

hdfs = HDFileSystem(host='trevally.amer.nevint.com', port=9000)

print hdfs.ls('/user/hadoop')

 def __init__(self, **kwargs):
     kwargs2 = {k: v for k, v in kwargs.items()
                if k in ['host', 'port', 'user', 'ticket_cache',
                         'token', 'pars']}
     HDFileSystem.__init__(self, connect=True, **kwargs2)

from tornado import gen

from dask.imperative import Value

from distributed.utils_test import gen_cluster, cluster, loop, make_hdfs
from distributed.utils import get_ip
from distributed.hdfs import (read_bytes, get_block_locations, write_bytes,
        _read_csv, read_csv)
from distributed import Executor
from distributed.executor import _wait, Future


pytest.importorskip('hdfs3')
from hdfs3 import HDFileSystem
try:
    hdfs = HDFileSystem(host='localhost', port=8020)
    hdfs.df()
    del hdfs
except:
    pytestmark = pytest.mark.skipif('True')


ip = get_ip()


def test_get_block_locations():
    with make_hdfs() as hdfs:
        data = b'a' * int(1e8)  # todo: reduce block size to speed up test
        fn_1 = '/tmp/test/file1'
        fn_2 = '/tmp/test/file2'

 def open(self, path, mode='rb', **kwargs):
     if path.startswith('hdfs://'):
         path = path[len('hdfs://'):]
     return HDFileSystem.open(self, path, mode, **kwargs)

def hdfs_open_file(path, auth):
    from hdfs3 import HDFileSystem
    hdfs = HDFileSystem(**auth)
    return hdfs.open(path, mode='rb')

    def open(self, path, mode='rb'):
        mode = mode.rstrip('b')
        return open(path, mode)

    def put(self, src, dst):
        return shutil.copy(src, dst)

if __name__ == "__main__":
    # load the hdfs node info
    f = open('hdfs.yml', 'r')
    data = yaml.load(f)
    f.close()

    hdfs_nn = data['hdfs_nn']
    hdfs = HDFileSystem(host=hdfs_nn, port=data['hdfs_port'])

    tfs = TransparentFileSystem(hdfs)
    print hdfs.exists('/tmp')
    # print hdfs.hoge('/tmp')
    print tfs.exists('/tmp')
    # print tfs.hoge('/tmp')

    # tfs_local = TransparentFileSystem()
    # print tfs_local.glob('/var/tmp')

    print 'test'
    print tfs.glob('/tmp')
    # tfs.hoge()
    tfs_local = TransparentFileSystem()
    # print tfs_local.glob('/home/vagrant/work/data/*')