Python PyWebHdfsClient.list_dir方法代码示例

# 需要导入模块: from pywebhdfs.webhdfs import PyWebHdfsClient [as 别名]
# 或者: from pywebhdfs.webhdfs.PyWebHdfsClient import list_dir [as 别名]
class HDFS(NDArray):
	'''
	HDFS storage

	Parameters
	----------
	name : str
		Name of directory to store text files (Path to the directory) without
		a leading '/'
	model : Model
		If None, the model is taken from the 'with' context
	vars : list of variables
		Sampling values will be stored for these variables. If None.
		'model.unobserved_RVs' is used
	host : str
		The IP address or hostname of the HDFS namenode. By default,
		it is 'localhost'
	port : str
		The port number for WebHDFS on the namenode. By default, it
		is '50070'
	user_name : str
		WebHDFS user_name used for authentication. By default, it is
		None
	'''
	def __init__(self, name, model=None, vars=None, host='localhost', port='50070', user_name=None):
		self.hdfs = PyWebHdfsClient(host=host, port=port, user_name=user_name)
		try:
			self.hdfs.list_dir(name)
		except FileNotFound:
			self.hdfs.make_dir(name)
		super(HDFS, self).__init__(name, model, vars)

	def close(self):
		super(HDFS, self).close()
		_dump_trace(self.name, self)

# 需要导入模块: from pywebhdfs.webhdfs import PyWebHdfsClient [as 别名]
# 或者: from pywebhdfs.webhdfs.PyWebHdfsClient import list_dir [as 别名]
class Store (store.Store):
    """
    HDFS backed store.
    """

    def __init__ (self):
        """ Connect to store """
        self._client = PyWebHdfsClient(host=store_host, port=store_port, user_name=store_user)

    def mkdir (self, path):
        self._client.make_dir(path)

    def read (self, path, open_handle):
        return StoreFile(self._client, path, "r", open_handle)

    def append (self, path, open_handle):
        return StoreFile(self._client, path, "a", open_handle)

    def write (self, path, open_handle):
        return StoreFile(self._client, path, "w", open_handle)

    def exists (self, path):
        try:
            dirinfo = self._client.list_dir(path)
            return True
        except errors.FileNotFound:
            return False
    
    def walk (self, path, visitor, recursive = False):
        """ Walk files in a path. Use recursive=True to include subdirs """
        dirinfo = self._client.list_dir(path)
        for status in dirinfo["FileStatuses"]["FileStatus"]:
            if recursive and status["type"] == "DIRECTORY":
                if len(path) > 0:
                    self.walk(path + "/" + status["pathSuffix"], visitor, recursive)
                else:
                    self.walk(status["pathSuffix"], visitor, recursive)
            else:
                info = dict(name=status["pathSuffix"], 
                            modify=datetime.fromtimestamp(status["modificationTime"]), 
                            size=status["length"])
                visitor(path, info)

# 需要导入模块: from pywebhdfs.webhdfs import PyWebHdfsClient [as 别名]
# 或者: from pywebhdfs.webhdfs.PyWebHdfsClient import list_dir [as 别名]
class WhenTestingListDirOperation(unittest.TestCase):

    def setUp(self):

        self.host = 'hostname'
        self.port = '00000'
        self.user_name = 'username'
        self.webhdfs = PyWebHdfsClient(host=self.host, port=self.port,
                                       user_name=self.user_name)
        self.response = MagicMock()
        self.requests = MagicMock(return_value=self.response)
        self.path = 'user/hdfs/old_dir'
        self.response = MagicMock()
        self.file_status = {
            "FileStatuses": {
                "FileStatus": [
                    {
                        "accessTime": 0,
                        "blockSize": 0,
                        "group": "supergroup",
                        "length": 24930,
                        "modificationTime": 1320173277227,
                        "owner": "webuser",
                        "pathSuffix": "a.patch",
                        "permission": "777",
                        "replication": 0,
                        "type": "FILE"
                    },
                    {
                        "accessTime": 0,
                        "blockSize": 0,
                        "group": "supergroup",
                        "length": 0,
                        "modificationTime": 1320173277227,
                        "owner": "webuser",
                        "pathSuffix": "",
                        "permission": "777",
                        "replication": 0,
                        "type": "DIRECTORY"
                    }
                ]
            }
        }
        self.response.json = MagicMock(return_value=self.file_status)

    def test_get_status_throws_exception_for_not_ok(self):

        self.response.status_code = httplib.BAD_REQUEST
        self.requests.get.return_value = self.response
        with patch('pywebhdfs.webhdfs.requests', self.requests):
            with self.assertRaises(errors.PyWebHdfsException):
                self.webhdfs.list_dir(self.path)

    def test_get_status_returns_true(self):

        self.response.status_code = httplib.OK
        self.requests.get.return_value = self.response
        with patch('pywebhdfs.webhdfs.requests', self.requests):
            result = self.webhdfs.list_dir(self.path)

        for key in result:
            self.assertEqual(result[key], self.file_status[key])

# 需要导入模块: from pywebhdfs.webhdfs import PyWebHdfsClient [as 别名]
# 或者: from pywebhdfs.webhdfs.PyWebHdfsClient import list_dir [as 别名]
# checksum reflects file changes
file_checksum = hdfs.get_file_checksum(example_file)
print(file_checksum)

# read in the data for the file
print('reading data from file at: {0}\n'.format(example_file))
file_data = hdfs.read_file(example_file)
print(file_data)

# rename the example_dir
print('renaming directory from {0} to {1}\n').format(example_dir, rename_dir)
hdfs.rename_file_dir(example_dir, '/{0}'.format(rename_dir))

# list the contents of the new directory
listdir_stats = hdfs.list_dir(rename_dir)
print(listdir_stats)

example_file = '{dir}/example.txt'.format(dir=rename_dir)

# delete the example file
print('deleting example file at: {0}'.format(example_file))
hdfs.delete_file_dir(example_file)

# list the contents of the directory
listdir_stats = hdfs.list_dir(rename_dir)
print(listdir_stats)

# delete the example directory
print('deleting the example directory at: {0}'.format(rename_dir))
hdfs.delete_file_dir(rename_dir, recursive='true')

# 需要导入模块: from pywebhdfs.webhdfs import PyWebHdfsClient [as 别名]
# 或者: from pywebhdfs.webhdfs.PyWebHdfsClient import list_dir [as 别名]
if __name__ == '__main__':
    global data_creator; #one object to hold all the data to stream to the client
    global date_list; #one list object to hold all the dates whose data are stored in HDFS
    #starting the new thread to run server separately
    server = Thread(target=runServer);
    server.start();
    #creating wall coordinates list for use later in this view
    DataCreator.read_transformed_coordinates_to_array();
    #instance that creates data in different format for each date
    data_creator =  DataCreator();  
    date_list=[];
     
    from pywebhdfs.webhdfs import PyWebHdfsClient;
    hdfs = PyWebHdfsClient(host='cshadoop.boisestate.edu',port='50070', user_name='uacharya');
    
    dir_listing = hdfs.list_dir('user/uacharya/flow')
    #list of dir dictionaries
    ls_dir = dir_listing['FileStatuses']['FileStatus']
    #appending to the list all the date which data is stored in hdfs
    for d in ls_dir:
        date_list.append(int(d['pathSuffix']))
        
    #creating required data in memory for all the data available   
    for date in date_list:  
        print("started creaing data for date {}".format(date))  
        data_creator.create_data_for_date(date);
        print(data_creator.check_available_data(date,aggregated=True))
        print(data_creator.check_available_data(date,bitmap=True))
    
    

# 需要导入模块: from pywebhdfs.webhdfs import PyWebHdfsClient [as 别名]
# 或者: from pywebhdfs.webhdfs.PyWebHdfsClient import list_dir [as 别名]
from os.path import join
from pywebhdfs.webhdfs import PyWebHdfsClient

def formatLine(line):
    line = line.values()
    line[0], line[5] = line[5], line[0]
    return ', '.join(line)

if __name__ == '__main__':

    host = 'hdfs://localhost:9000'
    ticker_path = host + '/user/hadoop/tickers.txt'
    save_path = host + '/user/hadoop/stock'

    hdfs = PyWebHdfsClient(host='localhost', port='50070', user_name='hadoop')
    folder = hdfs.list_dir('user/hadoop/stock')['FileStatuses']['FileStatus']
    files = sorted([dt.datetime.strptime(f['pathSuffix'].split('.')[0], '%Y-%m-%d').date() for f in folder])
    end = dt.date.today().strftime('%Y-%m-%d')
    
    sc = SparkContext(appName='stock_data')
    
    if len(files) > 3:
        hdfs.delete_file_dir(join(save_path, files[0].strftime('%Y-%m-%d') + '.csv'), recursive=True)

    if len(files) == 0:
        start = '2014-01-01'
        stockData = sc.textFile(ticker_path).flatMap(lambda x: Share(x).get_historical(start, end)).map(formatLine)
        stockData.saveAsTextFile(join(save_path, end + '.csv'))
    else:
        start = (files[-1] + dt.timedelta(days=1)).strftime('%Y-%m-%d')
        histStockData = sc.textFile(join(save_path, files[-1].strftime('%Y-%m-%d') + '.csv'))