当前位置: 首页>>代码示例>>Java>>正文

Java MRAppWithHistory类代码示例

本文整理汇总了Java中org.apache.hadoop.mapreduce.v2.hs.TestJobHistoryEvents.MRAppWithHistory类的典型用法代码示例。如果您正苦于以下问题：Java MRAppWithHistory类的具体用法？Java MRAppWithHistory怎么用？Java MRAppWithHistory使用的例子？那么恭喜您, 这里精选的类代码示例或许可以为您提供帮助。


MRAppWithHistory类属于org.apache.hadoop.mapreduce.v2.hs.TestJobHistoryEvents包，在下文中一共展示了MRAppWithHistory类的10个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Java代码示例。

示例1: testScanningOldDirs

import org.apache.hadoop.mapreduce.v2.hs.TestJobHistoryEvents.MRAppWithHistory; //导入依赖的package包/类
@Test(timeout = 50000)
public void testScanningOldDirs() throws Exception {
  LOG.info("STARTING testScanningOldDirs");
  try {
    Configuration conf = new Configuration();
    conf.setClass(
        NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
        MyResolver.class, DNSToSwitchMapping.class);
    RackResolver.init(conf);
    MRApp app = new MRAppWithHistory(1, 1, true, this.getClass().getName(),
        true);
    app.submit(conf);
    Job job = app.getContext().getAllJobs().values().iterator().next();
    JobId jobId = job.getID();
    LOG.info("JOBID is " + TypeConverter.fromYarn(jobId).toString());
    app.waitForState(job, JobState.SUCCEEDED);

    // make sure all events are flushed
    app.waitForState(Service.STATE.STOPPED);

    HistoryFileManagerForTest hfm = new HistoryFileManagerForTest();
    hfm.init(conf);
    HistoryFileInfo fileInfo = hfm.getFileInfo(jobId);
    Assert.assertNotNull("Unable to locate job history", fileInfo);

    // force the manager to "forget" the job
    hfm.deleteJobFromJobListCache(fileInfo);
    final int msecPerSleep = 10;
    int msecToSleep = 10 * 1000;
    while (fileInfo.isMovePending() && msecToSleep > 0) {
      Assert.assertTrue(!fileInfo.didMoveFail());
      msecToSleep -= msecPerSleep;
      Thread.sleep(msecPerSleep);
    }
    Assert.assertTrue("Timeout waiting for history move", msecToSleep > 0);

    fileInfo = hfm.getFileInfo(jobId);
    hfm.stop();
    Assert.assertNotNull("Unable to locate old job history", fileInfo);
    Assert.assertTrue("HistoryFileManager not shutdown properly",
        hfm.moveToDoneExecutor.isTerminated());
  } finally {
    LOG.info("FINISHED testScanningOldDirs");
  }
}
开发者ID:naver，项目名称:hadoop，代码行数:46，代码来源:TestJobHistoryParsing.java

示例2: testDeleteFileInfo

import org.apache.hadoop.mapreduce.v2.hs.TestJobHistoryEvents.MRAppWithHistory; //导入依赖的package包/类
/**
 * test clean old history files. Files should be deleted after 1 week by
 * default.
 */
@Test(timeout = 15000)
public void testDeleteFileInfo() throws Exception {
  LOG.info("STARTING testDeleteFileInfo");
  try {
    Configuration conf = new Configuration();
    conf.setClass(
        NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
        MyResolver.class, DNSToSwitchMapping.class);

    RackResolver.init(conf);
    MRApp app = new MRAppWithHistory(1, 1, true, this.getClass().getName(),
        true);
    app.submit(conf);
    Job job = app.getContext().getAllJobs().values().iterator().next();
    JobId jobId = job.getID();

    app.waitForState(job, JobState.SUCCEEDED);

    // make sure all events are flushed
    app.waitForState(Service.STATE.STOPPED);
    HistoryFileManager hfm = new HistoryFileManager();
    hfm.init(conf);
    HistoryFileInfo fileInfo = hfm.getFileInfo(jobId);
    hfm.initExisting();
    // wait for move files form the done_intermediate directory to the gone
    // directory
    while (fileInfo.isMovePending()) {
      Thread.sleep(300);
    }

    Assert.assertNotNull(hfm.jobListCache.values());

    // try to remove fileInfo
    hfm.clean();
    // check that fileInfo does not deleted
    Assert.assertFalse(fileInfo.isDeleted());
    // correct live time
    hfm.setMaxHistoryAge(-1);
    hfm.clean();
    hfm.stop();
    Assert.assertTrue("Thread pool shutdown",
        hfm.moveToDoneExecutor.isTerminated());
    // should be deleted !
    Assert.assertTrue("file should be deleted ", fileInfo.isDeleted());

  } finally {
    LOG.info("FINISHED testDeleteFileInfo");
  }
}
开发者ID:naver，项目名称:hadoop，代码行数:54，代码来源:TestJobHistoryParsing.java

示例3: testJobHistoryMethods

import org.apache.hadoop.mapreduce.v2.hs.TestJobHistoryEvents.MRAppWithHistory; //导入依赖的package包/类
/**
 * Simple test some methods of JobHistory
 */
@Test(timeout = 20000)
public void testJobHistoryMethods() throws Exception {
  LOG.info("STARTING testJobHistoryMethods");
  try {
    Configuration configuration = new Configuration();
    configuration
        .setClass(
            NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
            MyResolver.class, DNSToSwitchMapping.class);

    RackResolver.init(configuration);
    MRApp app = new MRAppWithHistory(1, 1, true, this.getClass().getName(),
        true);
    app.submit(configuration);
    Job job = app.getContext().getAllJobs().values().iterator().next();
    app.waitForState(job, JobState.SUCCEEDED);

    JobHistory jobHistory = new JobHistory();
    jobHistory.init(configuration);
    // Method getAllJobs
    Assert.assertEquals(1, jobHistory.getAllJobs().size());
    // and with ApplicationId
    Assert.assertEquals(1, jobHistory.getAllJobs(app.getAppID()).size());

    JobsInfo jobsinfo = jobHistory.getPartialJobs(0L, 10L, null, "default",
        0L, System.currentTimeMillis() + 1, 0L,
        System.currentTimeMillis() + 1, JobState.SUCCEEDED);

    Assert.assertEquals(1, jobsinfo.getJobs().size());
    Assert.assertNotNull(jobHistory.getApplicationAttemptId());
    // test Application Id
    Assert.assertEquals("application_0_0000", jobHistory.getApplicationID()
        .toString());
    Assert
        .assertEquals("Job History Server", jobHistory.getApplicationName());
    // method does not work
    Assert.assertNull(jobHistory.getEventHandler());
    // method does not work
    Assert.assertNull(jobHistory.getClock());
    // method does not work
    Assert.assertNull(jobHistory.getClusterInfo());

  } finally {
    LOG.info("FINISHED testJobHistoryMethods");
  }
}
开发者ID:naver，项目名称:hadoop，代码行数:50，代码来源:TestJobHistoryParsing.java

示例4: testDeleteFileInfo

import org.apache.hadoop.mapreduce.v2.hs.TestJobHistoryEvents.MRAppWithHistory; //导入依赖的package包/类
/**
 * Test clean old history files. Files should be deleted after 1 week by
 * default.
 */
@Test(timeout = 15000)
public void testDeleteFileInfo() throws Exception {
  LOG.info("STARTING testDeleteFileInfo");
  try {
    Configuration conf = new Configuration();

    conf.setClass(
        NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
        MyResolver.class, DNSToSwitchMapping.class);

    RackResolver.init(conf);
    MRApp app = new MRAppWithHistory(1, 1, true, this.getClass().getName(),
        true);
    app.submit(conf);
    Job job = app.getContext().getAllJobs().values().iterator().next();
    JobId jobId = job.getID();

    app.waitForState(job, JobState.SUCCEEDED);

    // make sure all events are flushed
    app.waitForState(Service.STATE.STOPPED);

    HistoryFileManager hfm = new HistoryFileManager();
    hfm.init(conf);
    HistoryFileInfo fileInfo = hfm.getFileInfo(jobId);
    hfm.initExisting();
    // wait for move files form the done_intermediate directory to the gone
    // directory
    while (fileInfo.isMovePending()) {
      Thread.sleep(300);
    }

    Assert.assertNotNull(hfm.jobListCache.values());

    // try to remove fileInfo
    hfm.clean();
    // check that fileInfo does not deleted
    Assert.assertFalse(fileInfo.isDeleted());
    // correct live time
    hfm.setMaxHistoryAge(-1);
    hfm.clean();
    hfm.stop();
    Assert.assertTrue("Thread pool shutdown",
        hfm.moveToDoneExecutor.isTerminated());
    // should be deleted !
    Assert.assertTrue("file should be deleted ", fileInfo.isDeleted());

  } finally {
    LOG.info("FINISHED testDeleteFileInfo");
  }
}
开发者ID:aliyun-beta，项目名称:aliyun-oss-hadoop-fs，代码行数:56，代码来源:TestJobHistoryParsing.java

示例5: testJobHistoryMethods

import org.apache.hadoop.mapreduce.v2.hs.TestJobHistoryEvents.MRAppWithHistory; //导入依赖的package包/类
/**
 * Simple test some methods of JobHistory
 */
@Test(timeout = 20000)
public void testJobHistoryMethods() throws Exception {
  LOG.info("STARTING testJobHistoryMethods");
  try {
    Configuration configuration = new Configuration();
    configuration
        .setClass(
            NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
            MyResolver.class, DNSToSwitchMapping.class);

    RackResolver.init(configuration);
    MRApp app = new MRAppWithHistory(1, 1, true, this.getClass().getName(),
        true);
    app.submit(configuration);
    Job job = app.getContext().getAllJobs().values().iterator().next();
    JobId jobId = job.getID();
    LOG.info("JOBID is " + TypeConverter.fromYarn(jobId).toString());
    app.waitForState(job, JobState.SUCCEEDED);

    JobHistory jobHistory = new JobHistory();
    jobHistory.init(configuration);
    // Method getAllJobs
    Assert.assertEquals(1, jobHistory.getAllJobs().size());
    // and with ApplicationId
    Assert.assertEquals(1, jobHistory.getAllJobs(app.getAppID()).size());

    JobsInfo jobsinfo = jobHistory.getPartialJobs(0L, 10L, null, "default",
        0L, System.currentTimeMillis() + 1, 0L,
        System.currentTimeMillis() + 1, JobState.SUCCEEDED);

    Assert.assertEquals(1, jobsinfo.getJobs().size());
    Assert.assertNotNull(jobHistory.getApplicationAttemptId());
    // test Application Id
    Assert.assertEquals("application_0_0000", jobHistory.getApplicationID()
        .toString());
    Assert
        .assertEquals("Job History Server", jobHistory.getApplicationName());
    // method does not work
    Assert.assertNull(jobHistory.getEventHandler());
    // method does not work
    Assert.assertNull(jobHistory.getClock());
    // method does not work
    Assert.assertNull(jobHistory.getClusterInfo());

  } finally {
    LOG.info("FINISHED testJobHistoryMethods");
  }
}
开发者ID:aliyun-beta，项目名称:aliyun-oss-hadoop-fs，代码行数:52，代码来源:TestJobHistoryParsing.java

示例6: testScanningOldDirs

import org.apache.hadoop.mapreduce.v2.hs.TestJobHistoryEvents.MRAppWithHistory; //导入依赖的package包/类
@Test(timeout = 50000)
public void testScanningOldDirs() throws Exception {
  LOG.info("STARTING testScanningOldDirs");
  try {
    Configuration conf = new Configuration();
    conf.setClass(
        CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
        MyResolver.class, DNSToSwitchMapping.class);
    RackResolver.init(conf);
    MRApp app = new MRAppWithHistory(1, 1, true, this.getClass().getName(),
        true);
    app.submit(conf);
    Job job = app.getContext().getAllJobs().values().iterator().next();
    JobId jobId = job.getID();
    LOG.info("JOBID is " + TypeConverter.fromYarn(jobId).toString());
    app.waitForState(job, JobState.SUCCEEDED);

    // make sure all events are flushed
    app.waitForState(Service.STATE.STOPPED);

    HistoryFileManagerForTest hfm = new HistoryFileManagerForTest();
    hfm.init(conf);
    HistoryFileInfo fileInfo = hfm.getFileInfo(jobId);
    Assert.assertNotNull("Unable to locate job history", fileInfo);

    // force the manager to "forget" the job
    hfm.deleteJobFromJobListCache(fileInfo);
    final int msecPerSleep = 10;
    int msecToSleep = 10 * 1000;
    while (fileInfo.isMovePending() && msecToSleep > 0) {
      Assert.assertTrue(!fileInfo.didMoveFail());
      msecToSleep -= msecPerSleep;
      Thread.sleep(msecPerSleep);
    }
    Assert.assertTrue("Timeout waiting for history move", msecToSleep > 0);

    fileInfo = hfm.getFileInfo(jobId);
    Assert.assertNotNull("Unable to locate old job history", fileInfo);
  } finally {
    LOG.info("FINISHED testScanningOldDirs");
  }
}
开发者ID:ict-carch，项目名称:hadoop-plus，代码行数:43，代码来源:TestJobHistoryParsing.java

示例7: testDeleteFileInfo

import org.apache.hadoop.mapreduce.v2.hs.TestJobHistoryEvents.MRAppWithHistory; //导入依赖的package包/类
/**
 * test clean old history files. Files should be deleted after 1 week by
 * default.
 */
@Test(timeout = 15000)
public void testDeleteFileInfo() throws Exception {
  LOG.info("STARTING testDeleteFileInfo");
  try {
    Configuration conf = new Configuration();
    conf.setClass(
        CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
        MyResolver.class, DNSToSwitchMapping.class);

    RackResolver.init(conf);
    MRApp app = new MRAppWithHistory(1, 1, true, this.getClass().getName(),
        true);
    app.submit(conf);
    Job job = app.getContext().getAllJobs().values().iterator().next();
    JobId jobId = job.getID();

    app.waitForState(job, JobState.SUCCEEDED);

    // make sure all events are flushed
    app.waitForState(Service.STATE.STOPPED);
    HistoryFileManager hfm = new HistoryFileManager();
    hfm.init(conf);
    HistoryFileInfo fileInfo = hfm.getFileInfo(jobId);
    hfm.initExisting();
    // wait for move files form the done_intermediate directory to the gone
    // directory
    while (fileInfo.isMovePending()) {
      Thread.sleep(300);
    }

    Assert.assertNotNull(hfm.jobListCache.values());

    // try to remove fileInfo
    hfm.clean();
    // check that fileInfo does not deleted
    Assert.assertFalse(fileInfo.isDeleted());
    // correct live time
    hfm.setMaxHistoryAge(-1);
    hfm.clean();
    // should be deleted !
    Assert.assertTrue("file should be deleted ", fileInfo.isDeleted());

  } finally {
    LOG.info("FINISHED testDeleteFileInfo");
  }
}
开发者ID:ict-carch，项目名称:hadoop-plus，代码行数:51，代码来源:TestJobHistoryParsing.java

示例8: testJobHistoryMethods

import org.apache.hadoop.mapreduce.v2.hs.TestJobHistoryEvents.MRAppWithHistory; //导入依赖的package包/类
/**
 * Simple test some methods of JobHistory
 */
@Test(timeout = 20000)
public void testJobHistoryMethods() throws Exception {
  LOG.info("STARTING testJobHistoryMethods");
  try {
    Configuration configuration = new Configuration();
    configuration
        .setClass(
            CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
            MyResolver.class, DNSToSwitchMapping.class);

    RackResolver.init(configuration);
    MRApp app = new MRAppWithHistory(1, 1, true, this.getClass().getName(),
        true);
    app.submit(configuration);
    Job job = app.getContext().getAllJobs().values().iterator().next();
    app.waitForState(job, JobState.SUCCEEDED);

    JobHistory jobHistory = new JobHistory();
    jobHistory.init(configuration);
    // Method getAllJobs
    Assert.assertEquals(1, jobHistory.getAllJobs().size());
    // and with ApplicationId
    Assert.assertEquals(1, jobHistory.getAllJobs(app.getAppID()).size());

    JobsInfo jobsinfo = jobHistory.getPartialJobs(0L, 10L, null, "default",
        0L, System.currentTimeMillis() + 1, 0L,
        System.currentTimeMillis() + 1, JobState.SUCCEEDED);

    Assert.assertEquals(1, jobsinfo.getJobs().size());
    Assert.assertNotNull(jobHistory.getApplicationAttemptId());
    // test Application Id
    Assert.assertEquals("application_0_0000", jobHistory.getApplicationID()
        .toString());
    Assert
        .assertEquals("Job History Server", jobHistory.getApplicationName());
    // method does not work
    Assert.assertNull(jobHistory.getEventHandler());
    // method does not work
    Assert.assertNull(jobHistory.getClock());
    // method does not work
    Assert.assertNull(jobHistory.getClusterInfo());

  } finally {
    LOG.info("FINISHED testJobHistoryMethods");
  }
}
开发者ID:ict-carch，项目名称:hadoop-plus，代码行数:50，代码来源:TestJobHistoryParsing.java

示例9: testScanningOldDirs

import org.apache.hadoop.mapreduce.v2.hs.TestJobHistoryEvents.MRAppWithHistory; //导入依赖的package包/类
@Test(timeout = 50000)
public void testScanningOldDirs() throws Exception {
  LOG.info("STARTING testScanningOldDirs");
  try {
    Configuration conf = new Configuration();
    conf.setClass(
        CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
        MyResolver.class, DNSToSwitchMapping.class);
    RackResolver.init(conf);
    MRApp app = new MRAppWithHistory(1, 1, true, this.getClass().getName(),
        true);
    app.submit(conf);
    Job job = app.getContext().getAllJobs().values().iterator().next();
    JobId jobId = job.getID();
    LOG.info("JOBID is " + TypeConverter.fromYarn(jobId).toString());
    app.waitForState(job, JobState.SUCCEEDED);

    // make sure all events are flushed
    app.waitForState(Service.STATE.STOPPED);

    HistoryFileManagerForTest hfm = new HistoryFileManagerForTest();
    hfm.init(conf);
    HistoryFileInfo fileInfo = hfm.getFileInfo(jobId);
    Assert.assertNotNull("Unable to locate job history", fileInfo);

    // force the manager to "forget" the job
    hfm.deleteJobFromJobListCache(fileInfo);
    final int msecPerSleep = 10;
    int msecToSleep = 10 * 1000;
    while (fileInfo.isMovePending() && msecToSleep > 0) {
      Assert.assertTrue(!fileInfo.didMoveFail());
      msecToSleep -= msecPerSleep;
      Thread.sleep(msecPerSleep);
    }
    Assert.assertTrue("Timeout waiting for history move", msecToSleep > 0);

    fileInfo = hfm.getFileInfo(jobId);
    hfm.stop();
    Assert.assertNotNull("Unable to locate old job history", fileInfo);
    Assert.assertTrue("HistoryFileManager not shutdown properly",
        hfm.moveToDoneExecutor.isTerminated());
  } finally {
    LOG.info("FINISHED testScanningOldDirs");
  }
}
开发者ID:chendave，项目名称:hadoop-TCP，代码行数:46，代码来源:TestJobHistoryParsing.java

示例10: testDeleteFileInfo

import org.apache.hadoop.mapreduce.v2.hs.TestJobHistoryEvents.MRAppWithHistory; //导入依赖的package包/类
/**
 * test clean old history files. Files should be deleted after 1 week by
 * default.
 */
@Test(timeout = 15000)
public void testDeleteFileInfo() throws Exception {
  LOG.info("STARTING testDeleteFileInfo");
  try {
    Configuration conf = new Configuration();
    conf.setClass(
        CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
        MyResolver.class, DNSToSwitchMapping.class);

    RackResolver.init(conf);
    MRApp app = new MRAppWithHistory(1, 1, true, this.getClass().getName(),
        true);
    app.submit(conf);
    Job job = app.getContext().getAllJobs().values().iterator().next();
    JobId jobId = job.getID();

    app.waitForState(job, JobState.SUCCEEDED);

    // make sure all events are flushed
    app.waitForState(Service.STATE.STOPPED);
    HistoryFileManager hfm = new HistoryFileManager();
    hfm.init(conf);
    HistoryFileInfo fileInfo = hfm.getFileInfo(jobId);
    hfm.initExisting();
    // wait for move files form the done_intermediate directory to the gone
    // directory
    while (fileInfo.isMovePending()) {
      Thread.sleep(300);
    }

    Assert.assertNotNull(hfm.jobListCache.values());

    // try to remove fileInfo
    hfm.clean();
    // check that fileInfo does not deleted
    Assert.assertFalse(fileInfo.isDeleted());
    // correct live time
    hfm.setMaxHistoryAge(-1);
    hfm.clean();
    hfm.stop();
    Assert.assertTrue("Thread pool shutdown",
        hfm.moveToDoneExecutor.isTerminated());
    // should be deleted !
    Assert.assertTrue("file should be deleted ", fileInfo.isDeleted());

  } finally {
    LOG.info("FINISHED testDeleteFileInfo");
  }
}
开发者ID:chendave，项目名称:hadoop-TCP，代码行数:54，代码来源:TestJobHistoryParsing.java


注：本文中的org.apache.hadoop.mapreduce.v2.hs.TestJobHistoryEvents.MRAppWithHistory类示例由纯净天空整理自Github/MSDocs等开源代码及文档管理平台，相关代码片段筛选自各路编程大神贡献的开源项目，源码版权归原作者所有，传播和使用请参考对应项目的License；未经允许，请勿转载。