本文整理汇总了Python中pywebhdfs.webhdfs.PyWebHdfsClient.list_dir方法的典型用法代码示例。如果您正苦于以下问题:Python PyWebHdfsClient.list_dir方法的具体用法?Python PyWebHdfsClient.list_dir怎么用?Python PyWebHdfsClient.list_dir使用的例子?那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类pywebhdfs.webhdfs.PyWebHdfsClient
的用法示例。
在下文中一共展示了PyWebHdfsClient.list_dir方法的6个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: HDFS
# 需要导入模块: from pywebhdfs.webhdfs import PyWebHdfsClient [as 别名]
# 或者: from pywebhdfs.webhdfs.PyWebHdfsClient import list_dir [as 别名]
class HDFS(NDArray):
'''
HDFS storage
Parameters
----------
name : str
Name of directory to store text files (Path to the directory) without
a leading '/'
model : Model
If None, the model is taken from the 'with' context
vars : list of variables
Sampling values will be stored for these variables. If None.
'model.unobserved_RVs' is used
host : str
The IP address or hostname of the HDFS namenode. By default,
it is 'localhost'
port : str
The port number for WebHDFS on the namenode. By default, it
is '50070'
user_name : str
WebHDFS user_name used for authentication. By default, it is
None
'''
def __init__(self, name, model=None, vars=None, host='localhost', port='50070', user_name=None):
self.hdfs = PyWebHdfsClient(host=host, port=port, user_name=user_name)
try:
self.hdfs.list_dir(name)
except FileNotFound:
self.hdfs.make_dir(name)
super(HDFS, self).__init__(name, model, vars)
def close(self):
super(HDFS, self).close()
_dump_trace(self.name, self)
示例2: Store
# 需要导入模块: from pywebhdfs.webhdfs import PyWebHdfsClient [as 别名]
# 或者: from pywebhdfs.webhdfs.PyWebHdfsClient import list_dir [as 别名]
class Store (store.Store):
"""
HDFS backed store.
"""
def __init__ (self):
""" Connect to store """
self._client = PyWebHdfsClient(host=store_host, port=store_port, user_name=store_user)
def mkdir (self, path):
self._client.make_dir(path)
def read (self, path, open_handle):
return StoreFile(self._client, path, "r", open_handle)
def append (self, path, open_handle):
return StoreFile(self._client, path, "a", open_handle)
def write (self, path, open_handle):
return StoreFile(self._client, path, "w", open_handle)
def exists (self, path):
try:
dirinfo = self._client.list_dir(path)
return True
except errors.FileNotFound:
return False
def walk (self, path, visitor, recursive = False):
""" Walk files in a path. Use recursive=True to include subdirs """
dirinfo = self._client.list_dir(path)
for status in dirinfo["FileStatuses"]["FileStatus"]:
if recursive and status["type"] == "DIRECTORY":
if len(path) > 0:
self.walk(path + "/" + status["pathSuffix"], visitor, recursive)
else:
self.walk(status["pathSuffix"], visitor, recursive)
else:
info = dict(name=status["pathSuffix"],
modify=datetime.fromtimestamp(status["modificationTime"]),
size=status["length"])
visitor(path, info)
示例3: WhenTestingListDirOperation
# 需要导入模块: from pywebhdfs.webhdfs import PyWebHdfsClient [as 别名]
# 或者: from pywebhdfs.webhdfs.PyWebHdfsClient import list_dir [as 别名]
class WhenTestingListDirOperation(unittest.TestCase):
def setUp(self):
self.host = 'hostname'
self.port = '00000'
self.user_name = 'username'
self.webhdfs = PyWebHdfsClient(host=self.host, port=self.port,
user_name=self.user_name)
self.response = MagicMock()
self.requests = MagicMock(return_value=self.response)
self.path = 'user/hdfs/old_dir'
self.response = MagicMock()
self.file_status = {
"FileStatuses": {
"FileStatus": [
{
"accessTime": 0,
"blockSize": 0,
"group": "supergroup",
"length": 24930,
"modificationTime": 1320173277227,
"owner": "webuser",
"pathSuffix": "a.patch",
"permission": "777",
"replication": 0,
"type": "FILE"
},
{
"accessTime": 0,
"blockSize": 0,
"group": "supergroup",
"length": 0,
"modificationTime": 1320173277227,
"owner": "webuser",
"pathSuffix": "",
"permission": "777",
"replication": 0,
"type": "DIRECTORY"
}
]
}
}
self.response.json = MagicMock(return_value=self.file_status)
def test_get_status_throws_exception_for_not_ok(self):
self.response.status_code = httplib.BAD_REQUEST
self.requests.get.return_value = self.response
with patch('pywebhdfs.webhdfs.requests', self.requests):
with self.assertRaises(errors.PyWebHdfsException):
self.webhdfs.list_dir(self.path)
def test_get_status_returns_true(self):
self.response.status_code = httplib.OK
self.requests.get.return_value = self.response
with patch('pywebhdfs.webhdfs.requests', self.requests):
result = self.webhdfs.list_dir(self.path)
for key in result:
self.assertEqual(result[key], self.file_status[key])
示例4: print
# 需要导入模块: from pywebhdfs.webhdfs import PyWebHdfsClient [as 别名]
# 或者: from pywebhdfs.webhdfs.PyWebHdfsClient import list_dir [as 别名]
# checksum reflects file changes
file_checksum = hdfs.get_file_checksum(example_file)
print(file_checksum)
# read in the data for the file
print('reading data from file at: {0}\n'.format(example_file))
file_data = hdfs.read_file(example_file)
print(file_data)
# rename the example_dir
print('renaming directory from {0} to {1}\n').format(example_dir, rename_dir)
hdfs.rename_file_dir(example_dir, '/{0}'.format(rename_dir))
# list the contents of the new directory
listdir_stats = hdfs.list_dir(rename_dir)
print(listdir_stats)
example_file = '{dir}/example.txt'.format(dir=rename_dir)
# delete the example file
print('deleting example file at: {0}'.format(example_file))
hdfs.delete_file_dir(example_file)
# list the contents of the directory
listdir_stats = hdfs.list_dir(rename_dir)
print(listdir_stats)
# delete the example directory
print('deleting the example directory at: {0}'.format(rename_dir))
hdfs.delete_file_dir(rename_dir, recursive='true')
示例5: Thread
# 需要导入模块: from pywebhdfs.webhdfs import PyWebHdfsClient [as 别名]
# 或者: from pywebhdfs.webhdfs.PyWebHdfsClient import list_dir [as 别名]
if __name__ == '__main__':
global data_creator; #one object to hold all the data to stream to the client
global date_list; #one list object to hold all the dates whose data are stored in HDFS
#starting the new thread to run server separately
server = Thread(target=runServer);
server.start();
#creating wall coordinates list for use later in this view
DataCreator.read_transformed_coordinates_to_array();
#instance that creates data in different format for each date
data_creator = DataCreator();
date_list=[];
from pywebhdfs.webhdfs import PyWebHdfsClient;
hdfs = PyWebHdfsClient(host='cshadoop.boisestate.edu',port='50070', user_name='uacharya');
dir_listing = hdfs.list_dir('user/uacharya/flow')
#list of dir dictionaries
ls_dir = dir_listing['FileStatuses']['FileStatus']
#appending to the list all the date which data is stored in hdfs
for d in ls_dir:
date_list.append(int(d['pathSuffix']))
#creating required data in memory for all the data available
for date in date_list:
print("started creaing data for date {}".format(date))
data_creator.create_data_for_date(date);
print(data_creator.check_available_data(date,aggregated=True))
print(data_creator.check_available_data(date,bitmap=True))
示例6: formatLine
# 需要导入模块: from pywebhdfs.webhdfs import PyWebHdfsClient [as 别名]
# 或者: from pywebhdfs.webhdfs.PyWebHdfsClient import list_dir [as 别名]
from os.path import join
from pywebhdfs.webhdfs import PyWebHdfsClient
def formatLine(line):
line = line.values()
line[0], line[5] = line[5], line[0]
return ', '.join(line)
if __name__ == '__main__':
host = 'hdfs://localhost:9000'
ticker_path = host + '/user/hadoop/tickers.txt'
save_path = host + '/user/hadoop/stock'
hdfs = PyWebHdfsClient(host='localhost', port='50070', user_name='hadoop')
folder = hdfs.list_dir('user/hadoop/stock')['FileStatuses']['FileStatus']
files = sorted([dt.datetime.strptime(f['pathSuffix'].split('.')[0], '%Y-%m-%d').date() for f in folder])
end = dt.date.today().strftime('%Y-%m-%d')
sc = SparkContext(appName='stock_data')
if len(files) > 3:
hdfs.delete_file_dir(join(save_path, files[0].strftime('%Y-%m-%d') + '.csv'), recursive=True)
if len(files) == 0:
start = '2014-01-01'
stockData = sc.textFile(ticker_path).flatMap(lambda x: Share(x).get_historical(start, end)).map(formatLine)
stockData.saveAsTextFile(join(save_path, end + '.csv'))
else:
start = (files[-1] + dt.timedelta(days=1)).strftime('%Y-%m-%d')
histStockData = sc.textFile(join(save_path, files[-1].strftime('%Y-%m-%d') + '.csv'))