本文整理汇总了Python中hdfs3.HDFileSystem.makedirs方法的典型用法代码示例。如果您正苦于以下问题：Python HDFileSystem.makedirs方法的具体用法？Python HDFileSystem.makedirs怎么用？Python HDFileSystem.makedirs使用的例子？那么, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类hdfs3.HDFileSystem的用法示例。
在下文中一共展示了HDFileSystem.makedirs方法的1个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: HadoopFileSystem
# 需要导入模块: from hdfs3 import HDFileSystem [as 别名]
# 或者: from hdfs3.HDFileSystem import makedirs [as 别名]
class HadoopFileSystem(FileSystem):
"""``FileSystem`` implementation that supports HDFS.
URL arguments to methods expect strings starting with ``hdfs://``.
Uses client library :class:`hdfs3.core.HDFileSystem`.
"""
def __init__(self):
"""Initializes a connection to HDFS.
Connection configuration is done using :doc:`hdfs`.
"""
super(HadoopFileSystem, self).__init__()
self._hdfs_client = HDFileSystem()
@classmethod
def scheme(cls):
return 'hdfs'
@staticmethod
def _parse_url(url):
"""Verifies that url begins with hdfs:// prefix, strips it and adds a
leading /.
Raises:
ValueError if url doesn't begin with hdfs://.
Args:
url: A URL in the form hdfs://path/...
Returns:
For an input of 'hdfs://path/...', will return '/path/...'.
"""
m = _URL_RE.match(url)
if m is None:
raise ValueError('Could not parse url: %s' % url)
return m.group(1)
def join(self, base_url, *paths):
"""Join two or more pathname components.
Args:
base_url: string path of the first component of the path.
Must start with hdfs://.
paths: path components to be added
Returns:
Full url after combining all the passed components.
"""
basepath = self._parse_url(base_url)
return _HDFS_PREFIX + self._join(basepath, *paths)
def _join(self, basepath, *paths):
return posixpath.join(basepath, *paths)
def split(self, url):
rel_path = self._parse_url(url)
head, tail = posixpath.split(rel_path)
return _HDFS_PREFIX + head, tail
def mkdirs(self, url):
path = self._parse_url(url)
if self._exists(path):
raise IOError('Path already exists: %s' % path)
return self._mkdirs(path)
def _mkdirs(self, path):
self._hdfs_client.makedirs(path)
def match(self, url_patterns, limits=None):
if limits is None:
limits = [None] * len(url_patterns)
if len(url_patterns) != len(limits):
raise BeamIOError(
'Patterns and limits should be equal in length: %d != %d' % (
len(url_patterns), len(limits)))
# TODO(udim): Update client to allow batched results.
def _match(path_pattern, limit):
"""Find all matching paths to the pattern provided."""
file_infos = self._hdfs_client.ls(path_pattern, detail=True)[:limit]
metadata_list = [FileMetadata(file_info['name'], file_info['size'])
for file_info in file_infos]
return MatchResult(path_pattern, metadata_list)
exceptions = {}
result = []
for url_pattern, limit in zip(url_patterns, limits):
try:
path_pattern = self._parse_url(url_pattern)
result.append(_match(path_pattern, limit))
except Exception as e: # pylint: disable=broad-except
exceptions[url_pattern] = e
if exceptions:
raise BeamIOError('Match operation failed', exceptions)
return result
#.........这里部分代码省略.........