本文整理汇总了Python中pywebhdfs.webhdfs.PyWebHdfsClient.make_dir方法的典型用法代码示例。如果您正苦于以下问题:Python PyWebHdfsClient.make_dir方法的具体用法?Python PyWebHdfsClient.make_dir怎么用?Python PyWebHdfsClient.make_dir使用的例子?那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类pywebhdfs.webhdfs.PyWebHdfsClient
的用法示例。
在下文中一共展示了PyWebHdfsClient.make_dir方法的11个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: HDFS
# 需要导入模块: from pywebhdfs.webhdfs import PyWebHdfsClient [as 别名]
# 或者: from pywebhdfs.webhdfs.PyWebHdfsClient import make_dir [as 别名]
class HDFS(NDArray):
'''
HDFS storage
Parameters
----------
name : str
Name of directory to store text files (Path to the directory) without
a leading '/'
model : Model
If None, the model is taken from the 'with' context
vars : list of variables
Sampling values will be stored for these variables. If None.
'model.unobserved_RVs' is used
host : str
The IP address or hostname of the HDFS namenode. By default,
it is 'localhost'
port : str
The port number for WebHDFS on the namenode. By default, it
is '50070'
user_name : str
WebHDFS user_name used for authentication. By default, it is
None
'''
def __init__(self, name, model=None, vars=None, host='localhost', port='50070', user_name=None):
self.hdfs = PyWebHdfsClient(host=host, port=port, user_name=user_name)
try:
self.hdfs.list_dir(name)
except FileNotFound:
self.hdfs.make_dir(name)
super(HDFS, self).__init__(name, model, vars)
def close(self):
super(HDFS, self).close()
_dump_trace(self.name, self)
示例2: WhenTestingMkdirOperation
# 需要导入模块: from pywebhdfs.webhdfs import PyWebHdfsClient [as 别名]
# 或者: from pywebhdfs.webhdfs.PyWebHdfsClient import make_dir [as 别名]
class WhenTestingMkdirOperation(unittest.TestCase):
def setUp(self):
self.host = 'hostname'
self.port = '00000'
self.user_name = 'username'
self.webhdfs = PyWebHdfsClient(host=self.host, port=self.port,
user_name=self.user_name)
self.response = MagicMock()
self.requests = MagicMock(return_value=self.response)
self.path = 'user/hdfs'
self.response = MagicMock()
def test_mkdir_throws_exception_for_not_ok(self):
self.response.status_code = httplib.BAD_REQUEST
self.requests.put.return_value = self.response
with patch('pywebhdfs.webhdfs.requests', self.requests):
with self.assertRaises(errors.PyWebHdfsException):
self.webhdfs.make_dir(self.path)
def test_mkdir_returns_true(self):
self.response.status_code = httplib.OK
self.requests.put.return_value = self.response
with patch('pywebhdfs.webhdfs.requests', self.requests):
result = self.webhdfs.make_dir(self.path)
self.assertTrue(result)
示例3: setup_common_oozie_libs
# 需要导入模块: from pywebhdfs.webhdfs import PyWebHdfsClient [as 别名]
# 或者: from pywebhdfs.webhdfs.PyWebHdfsClient import make_dir [as 别名]
def setup_common_oozie_libs(name_node):
webhdfs_port = '14000'
webhdfs_user = 'hdfs'
platform_dir = 'user/deployment/platform'
lib_path_list = ['/opt/cloudera/parcels/CDH/lib/hbase/hbase-client.jar',
'/opt/cloudera/parcels/CDH/lib/hbase/hbase-common.jar',
'/opt/cloudera/parcels/CDH/lib/hbase/hbase-protocol.jar',
'/opt/cloudera/parcels/CDH/lib/hbase/hbase-server.jar',
'/opt/cloudera/parcels/CDH/lib/hbase/lib/htrace-core.jar',
'/opt/cloudera/parcels/CDH/lib/hbase/hbase-hadoop-compat.jar',
'/opt/cloudera/parcels/CDH/lib/hbase/hbase-it.jar',
'/opt/cloudera/parcels/CDH/lib/hbase/hbase-prefix-tree.jar',
'/opt/cloudera/parcels/CDH/lib/hbase/lib/zookeeper.jar',
'/opt/cloudera/parcels/CDH/lib/pig/piggybank.jar',
'/opt/cloudera/parcels/CDH/lib/spark/lib/spark-examples.jar']
# Setup a connection with hdfs using namenode.
hdfs_client = PyWebHdfsClient(host=name_node, port=webhdfs_port, user_name=webhdfs_user, timeout=None)
# Create directory on hadoop file system (HDFS).
hdfs_client.make_dir(platform_dir)
# Creates a new file on HDFS and write contents from local FS.
for path in lib_path_list:
platform_file = '%s/%s' % (platform_dir, os.path.basename(path))
logging.info('Copying source file: %s to HDFS path %s', path, platform_file)
with open(path) as file_data:
hdfs_client.create_file(platform_file, file_data, overwrite=True)
示例4: setup_common_oozie_libs
# 需要导入模块: from pywebhdfs.webhdfs import PyWebHdfsClient [as 别名]
# 或者: from pywebhdfs.webhdfs.PyWebHdfsClient import make_dir [as 别名]
def setup_common_oozie_libs(name_node):
webhdfs_port = '14000'
webhdfs_user = 'hdfs'
platform_dir = 'user/deployment/platform'
lib_path_list = ['/usr/hdp/current/hbase-client/lib/hbase-client.jar',
'/usr/hdp/current/hbase-client/lib/hbase-common.jar',
'/usr/hdp/current/hbase-client/lib/hbase-protocol.jar',
'/usr/hdp/current/hbase-client/lib/hbase-server.jar',
'/usr/hdp/current/hbase-client/lib/htrace-core-3.1.0-incubating.jar',
'/usr/hdp/current/hbase-client/lib/hbase-hadoop-compat.jar',
'/usr/hdp/current/hbase-client/lib/hbase-it.jar',
'/usr/hdp/current/hbase-client/lib/hbase-prefix-tree.jar',
'/usr/hdp/current/hbase-client/lib/zookeeper.jar',
'/usr/hdp/current/pig-client/piggybank.jar',
'/usr/hdp/current/spark-client/lib/spark-examples.jar']
# Setup a connection with hdfs using namenode.
hdfs_client = PyWebHdfsClient(host=name_node, port=webhdfs_port, user_name=webhdfs_user, timeout=None)
# Create directory on hadoop file system (HDFS).
hdfs_client.make_dir(platform_dir)
# Creates a new file on HDFS and write contents from local FS.
for path in lib_path_list:
platform_file = '%s/%s' % (platform_dir, os.path.basename(path))
print 'Copying source file: %s to HDFS path %s' % (path, platform_file)
with open(path) as file_data:
try:
hdfs_client.create_file(platform_file, file_data, overwrite=True)
except PyWebHdfsException:
print 'retrying HDFS copy command for %s' % platform_file
time.sleep(5)
hdfs_client.create_file(platform_file, file_data, overwrite=True)
示例5: main
# 需要导入模块: from pywebhdfs.webhdfs import PyWebHdfsClient [as 别名]
# 或者: from pywebhdfs.webhdfs.PyWebHdfsClient import make_dir [as 别名]
def main(argv):
"""
Main method.
This method performs the following tasks:
1. Parse command line arguments
2. Retrieve credentials and connect to Cloudant and WebHDFS
3. Connect to the Cloudant `_changes` feed for checkpointed document
consumption
4. Process each change individually.
5. Upon exception throwing, store the latest checkpoint to local file and
exit.
"""
# add options into the parser
parser = configureOptions()
(options, args) = parser.parse_args()
checkRequiredArguments(options, parser)
print options
# configurations
last_seq = options.last_seq
# get credential
perm_file = "%s/.clou" % os.environ["HOME"]
creds = get_creds(perm_file)
# connect to source database
s = Server("https://%s:%[email protected]%s" % (creds["cloudant_user"], creds["cloudant_pwd"], options.uri))
db = s[options.dbname]
# print db.info()
# connect to target hdfs cluster
hdfs = PyWebHdfsClient(host=options.hdfs_host, port=options.hdfs_port, user_name=creds["hdfs_user"])
hdfs.make_dir(options.hdfs_path)
# and here we consume the cloudant `_changes` feed
counter = 0
changestream = ChangesStream(db, include_docs=True, heartbeat=True, since=last_seq)
for c in changestream:
# print c
try:
if counter % 100 == 0:
checkpoint(last_seq)
seq = processChange(hdfs, c, options.hdfs_path)
if seq: # protect against the last line being blank
last_seq = seq
counter += 1
except Exception:
traceback.print_exc()
checkpoint(last_seq)
os._exit(1)
checkpoint(last_seq)
示例6: Store
# 需要导入模块: from pywebhdfs.webhdfs import PyWebHdfsClient [as 别名]
# 或者: from pywebhdfs.webhdfs.PyWebHdfsClient import make_dir [as 别名]
class Store (store.Store):
"""
HDFS backed store.
"""
def __init__ (self):
""" Connect to store """
self._client = PyWebHdfsClient(host=store_host, port=store_port, user_name=store_user)
def mkdir (self, path):
self._client.make_dir(path)
def read (self, path, open_handle):
return StoreFile(self._client, path, "r", open_handle)
def append (self, path, open_handle):
return StoreFile(self._client, path, "a", open_handle)
def write (self, path, open_handle):
return StoreFile(self._client, path, "w", open_handle)
def exists (self, path):
try:
dirinfo = self._client.list_dir(path)
return True
except errors.FileNotFound:
return False
def walk (self, path, visitor, recursive = False):
""" Walk files in a path. Use recursive=True to include subdirs """
dirinfo = self._client.list_dir(path)
for status in dirinfo["FileStatuses"]["FileStatus"]:
if recursive and status["type"] == "DIRECTORY":
if len(path) > 0:
self.walk(path + "/" + status["pathSuffix"], visitor, recursive)
else:
self.walk(status["pathSuffix"], visitor, recursive)
else:
info = dict(name=status["pathSuffix"],
modify=datetime.fromtimestamp(status["modificationTime"]),
size=status["length"])
visitor(path, info)
示例7: PyWebHdfsClient
# 需要导入模块: from pywebhdfs.webhdfs import PyWebHdfsClient [as 别名]
# 或者: from pywebhdfs.webhdfs.PyWebHdfsClient import make_dir [as 别名]
_LOG = logging.getLogger(__name__)
example_dir = 'user/hdfs/example_dir'
example_file = '{dir}/example.txt'.format(dir=example_dir)
example_data = '01010101010101010101010101010101010101010101\n'
rename_dir = 'user/hdfs/example_rename'
# create a new client instance
hdfs = PyWebHdfsClient(host='localhost', port='50070',
user_name='hduser')
# create a new directory for the example
print('making new HDFS directory at: {0}\n'.format(example_dir))
hdfs.make_dir(example_dir)
# get a dictionary of the directory's status
dir_status = hdfs.get_file_dir_status(example_dir)
print(dir_status)
# create a new file on hdfs
print('making new file at: {0}\n'.format(example_file))
hdfs.create_file(example_file, example_data)
file_status = hdfs.get_file_dir_status(example_file)
print(file_status)
# get the checksum for the file
file_checksum = hdfs.get_file_checksum(example_file)
print(file_checksum)
示例8: PyWebHdfsClient
# 需要导入模块: from pywebhdfs.webhdfs import PyWebHdfsClient [as 别名]
# 或者: from pywebhdfs.webhdfs.PyWebHdfsClient import make_dir [as 别名]
#1 imports
from pywebhdfs.webhdfs import PyWebHdfsClient
#2 make connection with hadoop file system
hdfs = PyWebHdfsClient(user_name="hdfs",port=50070,host="sandbox.hortonworks.com")
hdfs.delete_file_dir('chapter5/LoanStats3d.csv',recursive=True)
#4 recreate the chapters directory
hdfs.make_dir('chapter5')
#5 upload the csv file
with open('./data/stored_csv.csv') as file_data:
hdfs.create_file('chapter5/LoanStats3d.csv',file_data, overwrite=True)
#6 print the status to see if this succeeded.
print hdfs.get_file_dir_status('chapter5/LoanStats3d.csv')
示例9: HDFS
# 需要导入模块: from pywebhdfs.webhdfs import PyWebHdfsClient [as 别名]
# 或者: from pywebhdfs.webhdfs.PyWebHdfsClient import make_dir [as 别名]
class HDFS(object):
def __init__(self, host, port, user):
self._hdfs = PyWebHdfsClient(
host=host, port=port, user_name=user, timeout=None)
logging.debug('webhdfs = %[email protected]%s:%s', user, host, port)
def recursive_copy(self, local_path, remote_path, exclude=None):
if exclude is None:
exclude = []
c_path = canonicalize(remote_path)
logging.debug('making %s', c_path)
self._hdfs.make_dir(c_path)
fs_g = os.walk(local_path)
for dpath, dnames, fnames in fs_g:
_, relative_path = dpath.split(local_path)
for dname in dnames:
if dname not in exclude:
c_path = canonicalize(
'%s/%s/%s' %
(remote_path, relative_path, dname))
logging.debug('making %s', c_path)
self._hdfs.make_dir(c_path)
for fname in fnames:
if fname not in exclude:
data = file(
canonicalize(
'%s/%s/%s' %
(local_path, relative_path, fname)), 'rb')
c_path = canonicalize(
'%s/%s/%s' %
(remote_path, relative_path, fname))
logging.debug('creating %s', c_path)
self._hdfs.create_file(c_path, data, overwrite=True)
data.close()
def make_dir(self, path):
logging.debug('make_dir: %s', path)
self._hdfs.make_dir(canonicalize(path))
def create_file(self, data, remote_file_path):
logging.debug('create_file: %s', remote_file_path)
sio = StringIO.StringIO(data)
self._hdfs.create_file(
canonicalize(remote_file_path),
sio,
overwrite=True)
def append_file(self, data, remote_file_path):
logging.debug('append to: %s', remote_file_path)
self._hdfs.append_file(canonicalize(remote_file_path), data)
def stream_file_to_disk(self, remote_file_path, local_file_path):
chunk_size = 10*1024*1024
offset = 0
with open(local_file_path, 'wb') as dest_file:
data = self._hdfs.read_file(canonicalize(remote_file_path), offset=offset, length=chunk_size)
while True:
dest_file.write(data)
if len(data) < chunk_size:
break
offset += chunk_size
data = self._hdfs.read_file(canonicalize(remote_file_path), offset=offset, length=chunk_size)
def read_file(self, remote_file_path):
data = self._hdfs.read_file(canonicalize(remote_file_path))
return data
def remove(self, path, recursive=False):
logging.debug('remove: %s', path)
self._hdfs.delete_file_dir(canonicalize(path), recursive)
示例10: save
# 需要导入模块: from pywebhdfs.webhdfs import PyWebHdfsClient [as 别名]
# 或者: from pywebhdfs.webhdfs.PyWebHdfsClient import make_dir [as 别名]
def save(self):
hdfs = PyWebHdfsClient(host=os.environ["WEBHDFS_HOST"], port='14000', user_name='oozie')
coordinator_path = "{0}/{1}/coordinator.xml".format(self.path, self.name)
hdfs.make_dir(self.path)
hdfs.create_file(coordinator_path, self.as_xml())
示例11: save
# 需要导入模块: from pywebhdfs.webhdfs import PyWebHdfsClient [as 别名]
# 或者: from pywebhdfs.webhdfs.PyWebHdfsClient import make_dir [as 别名]
def save(self, workflow_name="workflow.xml"):
hdfs = PyWebHdfsClient(host=os.environ["WEBHDFS_HOST"], port='14000', user_name='oozie')
workflow_path = "{0}/{1}/workflow.xml".format(self.path, self.name)
hdfs.make_dir(self.path)
hdfs.create_file(workflow_path, self.as_xml())