本文整理汇总了Python中mrjob.hadoop.HadoopJobRunner._find_probable_cause_of_failure方法的典型用法代码示例。如果您正苦于以下问题：Python HadoopJobRunner._find_probable_cause_of_failure方法的具体用法？Python HadoopJobRunner._find_probable_cause_of_failure怎么用？Python HadoopJobRunner._find_probable_cause_of_failure使用的例子？那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类mrjob.hadoop.HadoopJobRunner的用法示例。
在下文中一共展示了HadoopJobRunner._find_probable_cause_of_failure方法的1个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: FindProbableCauseOfFailureTestCase
# 需要导入模块: from mrjob.hadoop import HadoopJobRunner [as 别名]
# 或者: from mrjob.hadoop.HadoopJobRunner import _find_probable_cause_of_failure [as 别名]
class FindProbableCauseOfFailureTestCase(MockHadoopTestCase):
# integration tests for _find_probable_cause_of_failure()
def setUp(self):
super(FindProbableCauseOfFailureTestCase, self).setUp()
os.environ['MOCK_HADOOP_VERSION'] = '2.7.0'
self.runner = HadoopJobRunner()
def test_empty(self):
self.assertEqual(self.runner._find_probable_cause_of_failure(), None)
def test_yarn_python_exception(self):
APPLICATION_ID = 'application_1450486922681_0004'
CONTAINER_ID = 'container_1450486922681_0005_01_000003'
log_subdir = os.path.join(
os.environ['HADOOP_HOME'], 'logs',
'userlogs', APPLICATION_ID, CONTAINER_ID)
os.makedirs(log_subdir)
syslog_path = os.path.join(log_subdir, 'syslog')
with open(syslog_path, 'w') as syslog:
syslog.write(
'2015-12-21 14:06:17,707 INFO [main]'
' org.apache.hadoop.mapred.MapTask: Processing split:'
' hdfs://e4270474c8ee:9000/user/root/tmp/mrjob'
'/mr_boom.root.20151221.190511.059097/files'
'/bootstrap.sh:0+335\n')
syslog.write(
'2015-12-21 14:06:18,538 WARN [main]'
' org.apache.hadoop.mapred.YarnChild: Exception running child'
' : java.lang.RuntimeException:'
' PipeMapRed.waitOutputThreads(): subprocess failed with'
' code 1\n')
syslog.write(
' at org.apache.hadoop.streaming.PipeMapRed'
'.waitOutputThreads(PipeMapRed.java:322)\n')
stderr_path = os.path.join(log_subdir, 'stderr')
with open(stderr_path, 'w') as stderr:
stderr.write('Traceback (most recent call last):\n')
stderr.write(' File "mr_boom.py", line 10, in <module>\n')
stderr.write(' MRBoom.run()\n')
stderr.write('Exception: BOOM\n')
# need application_id
self.assertIsNone(self.runner._find_probable_cause_of_failure())
cause = self.runner._find_probable_cause_of_failure(
application_id=APPLICATION_ID)
self.assertTrue(cause)
self.assertEqual(cause['syslog']['path'], syslog_path)
self.assertTrue(cause['syslog']['error'])
self.assertEqual(cause['stderr']['path'], stderr_path)
self.assertTrue(cause['stderr']['error'])