本文整理汇总了Python中mrjob.hadoop.HadoopJobRunner._hadoop_log_dirs方法的典型用法代码示例。如果您正苦于以下问题：Python HadoopJobRunner._hadoop_log_dirs方法的具体用法？Python HadoopJobRunner._hadoop_log_dirs怎么用？Python HadoopJobRunner._hadoop_log_dirs使用的例子？那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类mrjob.hadoop.HadoopJobRunner的用法示例。
在下文中一共展示了HadoopJobRunner._hadoop_log_dirs方法的1个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: HadoopLogDirsTestCase
# 需要导入模块: from mrjob.hadoop import HadoopJobRunner [as 别名]
# 或者: from mrjob.hadoop.HadoopJobRunner import _hadoop_log_dirs [as 别名]
class HadoopLogDirsTestCase(SandboxedTestCase):
def setUp(self):
super(HadoopLogDirsTestCase, self).setUp()
os.environ.clear()
self.mock_hadoop_version = '2.7.0'
# the result of _hadoop_dir(). This handles non-log-specific
# environment variables, such as $HADOOP_PREFIX, and also guesses
# based on the path of the Hadoop binary
self.mock_hadoop_dirs = []
def mock_get_hadoop_version():
return self.mock_hadoop_version
def mock_hadoop_dirs_method():
return (d for d in self.mock_hadoop_dirs)
self.start(patch('mrjob.hadoop.HadoopJobRunner.get_hadoop_version',
side_effect=mock_get_hadoop_version))
self.start(patch('mrjob.hadoop.HadoopJobRunner._hadoop_dirs',
side_effect=mock_hadoop_dirs_method))
self.runner = HadoopJobRunner()
def test_empty(self):
self.assertEqual(list(self.runner._hadoop_log_dirs()),
['hdfs:///tmp/hadoop-yarn/staging',
'/mnt/var/log/hadoop'])
def test_precedence(self):
os.environ['HADOOP_LOG_DIR'] = '/path/to/hadoop-log-dir'
os.environ['YARN_LOG_DIR'] = '/path/to/yarn-log-dir'
self.mock_hadoop_dirs = ['/path/to/hadoop-prefix',
'/path/to/hadoop-home']
self.assertEqual(
list(self.runner._hadoop_log_dirs(output_dir='hdfs:///output/')),
['/path/to/hadoop-log-dir',
'/path/to/yarn-log-dir',
'hdfs:///tmp/hadoop-yarn/staging',
'hdfs:///output/_logs',
'/path/to/hadoop-prefix/logs',
'/path/to/hadoop-home/logs',
'/mnt/var/log/hadoop'])
def test_hadoop_log_dirs_opt(self):
self.runner = HadoopJobRunner(hadoop_log_dirs=['/logs1', '/logs2'])
os.environ['HADOOP_LOG_DIR'] = '/path/to/hadoop-log-dir'
# setting hadoop_log_dirs short-circuits automatic discovery of logs
self.assertEqual(
list(self.runner._hadoop_log_dirs()),
['/logs1', '/logs2'])
def test_need_yarn_for_yarn_log_dir_and_hdfs_log_dir(self):
os.environ['YARN_LOG_DIR'] = '/path/to/yarn-log-dir'
self.mock_hadoop_version = '2.0.0'
self.assertEqual(list(self.runner._hadoop_log_dirs()),
['/path/to/yarn-log-dir',
'hdfs:///tmp/hadoop-yarn/staging',
'/mnt/var/log/hadoop'])
self.mock_hadoop_version = '1.0.3'
self.assertEqual(list(self.runner._hadoop_log_dirs()),
['/mnt/var/log/hadoop'])