本文整理汇总了Python中mrjob.hadoop.HadoopJobRunner._find_hadoop_streaming_jar方法的典型用法代码示例。如果您正苦于以下问题：Python HadoopJobRunner._find_hadoop_streaming_jar方法的具体用法？Python HadoopJobRunner._find_hadoop_streaming_jar怎么用？Python HadoopJobRunner._find_hadoop_streaming_jar使用的例子？那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类mrjob.hadoop.HadoopJobRunner的用法示例。
在下文中一共展示了HadoopJobRunner._find_hadoop_streaming_jar方法的1个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: HadoopStreamingJarTestCase
# 需要导入模块: from mrjob.hadoop import HadoopJobRunner [as 别名]
# 或者: from mrjob.hadoop.HadoopJobRunner import _find_hadoop_streaming_jar [as 别名]
class HadoopStreamingJarTestCase(SandboxedTestCase):
def setUp(self):
super(HadoopStreamingJarTestCase, self).setUp()
self.mock_paths = []
def mock_ls(path): # don't bother to support globs
return (p for p in sorted(self.mock_paths) if p.startswith(path))
self.start(patch('mrjob.fs.local.LocalFilesystem.ls',
side_effect=mock_ls))
os.environ.clear()
self.runner = HadoopJobRunner()
def test_empty_fs(self):
self.assertEqual(self.runner._find_hadoop_streaming_jar(), None)
def test_deprecated_hadoop_home_option(self):
self.runner = HadoopJobRunner(hadoop_home='/ha/do/op/home-option')
self.mock_paths.append('/ha/do/op/home-option/hadoop-streaming.jar')
self.assertEqual(self.runner._find_hadoop_streaming_jar(),
'/ha/do/op/home-option/hadoop-streaming.jar')
def test_deprecated_hadoop_home_option_beats_hadoop_prefix(self):
os.environ['HADOOP_PREFIX'] = '/ha/do/op/prefix'
self.mock_paths.append('/ha/do/op/prefix/hadoop-streaming.jar')
self.test_deprecated_hadoop_home_option()
# tests of well-known environment variables
def test_hadoop_prefix(self):
os.environ['HADOOP_PREFIX'] = '/ha/do/op/prefix'
self.mock_paths.append('/ha/do/op/prefix/hadoop-streaming.jar')
self.assertEqual(self.runner._find_hadoop_streaming_jar(),
'/ha/do/op/prefix/hadoop-streaming.jar')
def test_hadoop_prefix_beats_hadoop_home(self):
os.environ['HADOOP_HOME'] = '/ha/do/op/home'
self.mock_paths.append('/ha/do/op/home/hadoop-streaming.jar')
self.test_hadoop_prefix()
def test_hadoop_home(self):
os.environ['HADOOP_HOME'] = '/ha/do/op/home'
self.mock_paths.append('/ha/do/op/home/hadoop-streaming.jar')
self.assertEqual(self.runner._find_hadoop_streaming_jar(),
'/ha/do/op/home/hadoop-streaming.jar')
def test_hadoop_home_beats_hadoop_install(self):
os.environ['HADOOP_INSTALL'] = '/ha/do/op/install'
self.mock_paths.append('/ha/do/op/install/hadoop-streaming.jar')
self.test_hadoop_home()
def test_hadoop_install(self):
os.environ['HADOOP_INSTALL'] = '/ha/do/op/install'
self.mock_paths.append('/ha/do/op/install/hadoop-streaming.jar')
self.assertEqual(self.runner._find_hadoop_streaming_jar(),
'/ha/do/op/install/hadoop-streaming.jar')
def test_hadoop_install_beats_hadoop_mapred_home(self):
os.environ['HADOOP_MAPRED_HOME'] = '/ha/do/op/mapred-home'
self.mock_paths.append('/ha/do/op/mapred-home/hadoop-streaming.jar')
self.test_hadoop_install()
def test_hadoop_mapred_home(self):
os.environ['HADOOP_MAPRED_HOME'] = '/ha/do/op/mapred-home'
self.mock_paths.append('/ha/do/op/mapred-home/hadoop-streaming.jar')
self.assertEqual(self.runner._find_hadoop_streaming_jar(),
'/ha/do/op/mapred-home/hadoop-streaming.jar')
def test_hadoop_mapred_home_beats_infer_from_hadoop_bin(self):
self.runner = HadoopJobRunner(
hadoop_bin=['/ha/do/op/bin-parent/bin/hadoop'])
self.mock_paths.append('/ha/do/op/bin-parent/hadoop-streaming.jar')
self.test_hadoop_mapred_home()
# infer from hadoop_bin
def test_infer_from_hadoop_bin_parent_dir(self):
self.runner = HadoopJobRunner(
hadoop_bin=['/ha/do/op/bin-parent/bin/hadoop'])
self.mock_paths.append('/ha/do/op/bin-parent/hadoop-streaming.jar')
self.assertEqual(self.runner._find_hadoop_streaming_jar(),
'/ha/do/op/bin-parent/hadoop-streaming.jar')
def test_hadoop_bin_beats_hadoop_anything_home(self):
#.........这里部分代码省略.........