Java GiraphConfiguration.setEventWaitMsecs方法代码示例

import org.apache.giraph.conf.GiraphConfiguration; //导入方法依赖的package包/类
/**
 * Run a job that requires checkpointing and will have a worker crash
 * and still recover from a previous checkpoint.
 *
 * @throws IOException
 * @throws ClassNotFoundException
 * @throws InterruptedException
 */
@Test
public void testSingleFault()
  throws IOException, InterruptedException, ClassNotFoundException {
  if (!runningInDistributedMode()) {
    System.out.println(
        "testSingleFault: Ignore this test in local mode.");
    return;
  }
  Path outputPath = getTempPath(getCallingMethodName());
  GiraphConfiguration conf = new GiraphConfiguration();
  conf.setComputationClass(
      SimpleCheckpoint.SimpleCheckpointComputation.class);
  conf.setWorkerContextClass(
      SimpleCheckpoint.SimpleCheckpointVertexWorkerContext.class);
  conf.setMasterComputeClass(
      SimpleCheckpoint.SimpleCheckpointVertexMasterCompute.class);
  conf.setVertexInputFormatClass(SimpleSuperstepVertexInputFormat.class);
  conf.setVertexOutputFormatClass(SimpleSuperstepVertexOutputFormat.class);
  conf.setBoolean(SimpleCheckpoint.ENABLE_FAULT, true);
  conf.setInt("mapred.map.max.attempts", 4);
  // Trigger failure faster
  conf.setInt("mapred.task.timeout", 10000);
  conf.setMaxMasterSuperstepWaitMsecs(10000);
  conf.setEventWaitMsecs(1000);
  conf.setCheckpointFrequency(2);
  GiraphConstants.CHECKPOINT_DIRECTORY.set(conf,
      getTempPath("_singleFaultCheckpoints").toString());
  GiraphConstants.CLEANUP_CHECKPOINTS_AFTER_SUCCESS.set(conf, false);
  GiraphConstants.ZOOKEEPER_SESSION_TIMEOUT.set(conf, 10000);
  GiraphConstants.ZOOKEEPER_MIN_SESSION_TIMEOUT.set(conf, 10000);
  GiraphJob job = prepareJob(getCallingMethodName(), conf, outputPath);
  assertTrue(job.run(true));
}
 

import org.apache.giraph.conf.GiraphConfiguration; //导入方法依赖的package包/类
/**
 * Set up the GiraphConfiguration settings we need to run a no-op Giraph
 * job on a MiniYARNCluster as an integration test. Some YARN-specific
 * flags are set inside GiraphYarnClient and won't need to be set here.
 */
private void setupYarnConfiguration() throws IOException {
  conf = new GiraphConfiguration();
  conf.setWorkerConfiguration(1, 1, 100.0f);
  conf.setMaxMasterSuperstepWaitMsecs(30 * 1000);
  conf.setEventWaitMsecs(3 * 1000);
  conf.setYarnLibJars(""); // no need
  conf.setYarnTaskHeapMb(256); // small since no work to be done
  conf.setComputationClass(DummyYarnComputation.class);
  conf.setVertexInputFormatClass(IntIntNullTextInputFormat.class);
  conf.setVertexOutputFormatClass(IdWithValueTextOutputFormat.class);
  conf.setNumComputeThreads(1);
  conf.setMaxTaskAttempts(1);
  conf.setNumInputSplitsThreads(1);
  // Giraph on YARN only ever things its running in "non-local" mode
  conf.setLocalTestMode(false);
  // this has to happen here before we populate the conf with the temp dirs
  setupTempDirectories();
  conf.set(OUTDIR, new Path(outputDir.getAbsolutePath()).toString());
  GiraphFileInputFormat.addVertexInputPath(conf, new Path(inputDir.getAbsolutePath()));
  // hand off the ZK info we just created to our no-op job
  GiraphConstants.ZOOKEEPER_SERVERLIST_POLL_MSECS.set(conf, 500);
  conf.setZooKeeperConfiguration(zkList);
  conf.set(GiraphConstants.ZOOKEEPER_DIR, zkDir.getAbsolutePath());
  GiraphConstants.ZOOKEEPER_MANAGER_DIRECTORY.set(conf, zkMgrDir.getAbsolutePath());
  // without this, our "real" client won't connect w/"fake" YARN cluster
  conf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_FIXED_PORTS, true);
}
 

import org.apache.giraph.conf.GiraphConfiguration; //导入方法依赖的package包/类
/**
 * Run a job that requires checkpointing and will have a worker crash
 * and still recover from a previous checkpoint.
 *
 * @throws IOException
 * @throws ClassNotFoundException
 * @throws InterruptedException
 */
@Test
public void testSingleFault()
  throws IOException, InterruptedException, ClassNotFoundException {
  if (!runningInDistributedMode()) {
    System.out.println(
        "testSingleFault: Ignore this test in local mode.");
    return;
  }
  Path outputPath = getTempPath(getCallingMethodName());
  GiraphConfiguration conf = new GiraphConfiguration();
  conf.setVertexClass(
      SimpleCheckpointVertex.SimpleCheckpointComputation.class);
  conf.setWorkerContextClass(
      SimpleCheckpointVertex.SimpleCheckpointVertexWorkerContext.class);
  conf.setMasterComputeClass(
      SimpleCheckpointVertex.SimpleCheckpointVertexMasterCompute.class);
  conf.setVertexInputFormatClass(SimpleSuperstepVertexInputFormat.class);
  conf.setVertexOutputFormatClass(SimpleSuperstepVertexOutputFormat.class);
  conf.setBoolean(SimpleCheckpointVertex.ENABLE_FAULT, true);
  conf.setInt("mapred.map.max.attempts", 4);
  // Trigger failure faster
  conf.setInt("mapred.task.timeout", 10000);
  conf.setMaxMasterSuperstepWaitMsecs(10000);
  conf.setEventWaitMsecs(1000);
  conf.setCheckpointFrequency(2);
  GiraphConstants.CHECKPOINT_DIRECTORY.set(conf,
      getTempPath("_singleFaultCheckpoints").toString());
  GiraphConstants.CLEANUP_CHECKPOINTS_AFTER_SUCCESS.set(conf, false);
  GiraphConstants.ZOOKEEPER_SESSION_TIMEOUT.set(conf, 10000);
  GiraphConstants.ZOOKEEPER_MIN_SESSION_TIMEOUT.set(conf, 10000);
  GiraphJob job = prepareJob(getCallingMethodName(), conf, outputPath);
  assertTrue(job.run(true));
}
 

import org.apache.giraph.conf.GiraphConfiguration; //导入方法依赖的package包/类
/**
 * Set up the GiraphConfiguration settings we need to run a no-op Giraph
 * job on a MiniYARNCluster as an integration test. Some YARN-specific
 * flags are set inside GiraphYarnClient and won't need to be set here.
 */
private void setupYarnConfiguration() throws IOException {
  conf = new GiraphConfiguration();
  conf.setWorkerConfiguration(1, 1, 100.0f);
  conf.setMaxMasterSuperstepWaitMsecs(30 * 1000);
  conf.setEventWaitMsecs(3 * 1000);
  conf.setYarnLibJars(""); // no need
  conf.setYarnTaskHeapMb(256); // small since no work to be done
  conf.setVertexClass(DummyYarnVertex.class);
  conf.setVertexInputFormatClass(IntIntNullTextInputFormat.class);
  conf.setVertexOutputFormatClass(IdWithValueTextOutputFormat.class);
  conf.setNumComputeThreads(1);
  conf.setMaxTaskAttempts(1);
  conf.setNumInputSplitsThreads(1);
  // Giraph on YARN only ever things its running in "non-local" mode
  conf.setLocalTestMode(false);
  // this has to happen here before we populate the conf with the temp dirs
  setupTempDirectories();
  conf.set(OUTDIR, new Path(outputDir.getAbsolutePath()).toString());
  GiraphFileInputFormat.addVertexInputPath(conf, new Path(inputDir.getAbsolutePath()));
  // hand off the ZK info we just created to our no-op job
  GiraphConstants.ZOOKEEPER_SERVERLIST_POLL_MSECS.set(conf, 500);
  conf.setZooKeeperConfiguration(zkList);
  conf.set(GiraphConstants.ZOOKEEPER_DIR, zkDir.getAbsolutePath());
  GiraphConstants.ZOOKEEPER_MANAGER_DIRECTORY.set(conf, zkMgrDir.getAbsolutePath());
  // without this, our "real" client won't connect w/"fake" YARN cluster
  conf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_FIXED_PORTS, true);
}
 

import org.apache.giraph.conf.GiraphConfiguration; //导入方法依赖的package包/类
/**
 * Adjust the configuration to the basic test case
 */
public final Configuration setupConfiguration(GiraphJob job)
    throws IOException {
  GiraphConfiguration conf = job.getConfiguration();
  conf.set("mapred.jar", getJarLocation());

  // Allow this test to be run on a real Hadoop setup
  if (runningInDistributedMode()) {
    System.out.println("setupConfiguration: Sending job to job tracker " +
        jobTracker + " with jar path " + getJarLocation()
        + " for " + getName());
    conf.set("mapred.job.tracker", jobTracker);
    conf.setWorkerConfiguration(getNumWorkers(), getNumWorkers(), 100.0f);
  }
  else {
    System.out.println("setupConfiguration: Using local job runner with " +
        "location " + getJarLocation() + " for " + getName());
    conf.setWorkerConfiguration(1, 1, 100.0f);
    // Single node testing
    GiraphConstants.SPLIT_MASTER_WORKER.set(conf, false);
    GiraphConstants.LOCAL_TEST_MODE.set(conf, true);
  }
  conf.setMaxMasterSuperstepWaitMsecs(30 * 1000);
  conf.setEventWaitMsecs(3 * 1000);
  GiraphConstants.ZOOKEEPER_SERVERLIST_POLL_MSECS.set(conf, 500);
  if (getZooKeeperList() != null) {
    conf.setZooKeeperConfiguration(getZooKeeperList());
  }
  // GeneratedInputSplit will generate 5 vertices
  conf.setLong(READER_VERTICES_OPT, 5);

  // Setup pathes for temporary files
  Path zookeeperDir = getTempPath("_bspZooKeeper");
  Path zkManagerDir = getTempPath("_defaultZkManagerDir");
  Path checkPointDir = getTempPath("_checkpoints");

  // We might start several jobs per test, so we need to clean up here
  FileUtils.deletePath(conf, zookeeperDir);
  FileUtils.deletePath(conf, zkManagerDir);
  FileUtils.deletePath(conf, checkPointDir);

  conf.set(GiraphConstants.ZOOKEEPER_DIR, zookeeperDir.toString());
  GiraphConstants.ZOOKEEPER_MANAGER_DIRECTORY.set(conf,
      zkManagerDir.toString());
  GiraphConstants.CHECKPOINT_DIRECTORY.set(conf, checkPointDir.toString());

  return conf;
}
 

import org.apache.giraph.conf.GiraphConfiguration; //导入方法依赖的package包/类
/**
 * Adjust the configuration to the basic test case
 */
public final Configuration setupConfiguration(GiraphJob job)
    throws IOException {
  GiraphConfiguration conf = job.getConfiguration();
  conf.set("mapred.jar", getJarLocation());

  // Allow this test to be run on a real Hadoop setup
  if (runningInDistributedMode()) {
    System.out.println("setupConfiguration: Sending job to job tracker " +
        jobTracker + " with jar path " + getJarLocation()
        + " for " + getName());
    conf.set("mapred.job.tracker", jobTracker);
    conf.setWorkerConfiguration(getNumWorkers(), getNumWorkers(), 100.0f);
  }
  else {
    System.out.println("setupConfiguration: Using local job runner with " +
        "location " + getJarLocation() + " for " + getName());
    conf.setWorkerConfiguration(1, 1, 100.0f);
    // Single node testing
    GiraphConstants.SPLIT_MASTER_WORKER.set(conf, false);
  }
  conf.setMaxMasterSuperstepWaitMsecs(30 * 1000);
  conf.setEventWaitMsecs(3 * 1000);
  GiraphConstants.ZOOKEEPER_SERVERLIST_POLL_MSECS.set(conf, 500);
  if (getZooKeeperList() != null) {
    conf.setZooKeeperConfiguration(getZooKeeperList());
  }
  // GeneratedInputSplit will generate 5 vertices
  conf.setLong(READER_VERTICES, 5);

  // Setup pathes for temporary files
  Path zookeeperDir = getTempPath("_bspZooKeeper");
  Path zkManagerDir = getTempPath("_defaultZkManagerDir");
  Path checkPointDir = getTempPath("_checkpoints");

  // We might start several jobs per test, so we need to clean up here
  FileUtils.deletePath(conf, zookeeperDir);
  FileUtils.deletePath(conf, zkManagerDir);
  FileUtils.deletePath(conf, checkPointDir);

  conf.set(GiraphConstants.ZOOKEEPER_DIR, zookeeperDir.toString());
  GiraphConstants.ZOOKEEPER_MANAGER_DIRECTORY.set(conf,
      zkManagerDir.toString());
  GiraphConstants.CHECKPOINT_DIRECTORY.set(conf, checkPointDir.toString());

  return conf;
}