Java GiraphJob类代码示例

import org.apache.giraph.job.GiraphJob; //导入依赖的package包/类
/**
 * Run the ZooKeeper in-process and the job.
 *
 * @param zookeeperConfig Quorum peer configuration
 * @param giraphJob       Giraph job to run
 * @return True if successful, false otherwise
 */
private static boolean runZooKeeperAndJob(
        final ZookeeperConfig zookeeperConfig,
        GiraphJob giraphJob) throws IOException {
    final InProcessZooKeeperRunner.ZooKeeperServerRunner zookeeper =
            new InProcessZooKeeperRunner.ZooKeeperServerRunner();

    int port = zookeeper.start(zookeeperConfig);

    LOG.info("Started test zookeeper on port " + port);
    GiraphConstants.ZOOKEEPER_LIST.set(giraphJob.getConfiguration(),
            "localhost:" + port);
    try {
        return giraphJob.run(true);
    } catch (InterruptedException |
            ClassNotFoundException | IOException e) {
        LOG.error("runZooKeeperAndJob: Got exception on running", e);
    } finally {
        zookeeper.stop();
    }

    return false;
}
 

import org.apache.giraph.job.GiraphJob; //导入依赖的package包/类
@Override
public int run(String[] args) throws Exception {
	if (null == getConf()) {
		conf = new Configuration();
	}
	GiraphConfiguration giraphConf = new GiraphConfiguration(getConf());
	CommandLine cmd = ConfigurationUtils.parseArgs(giraphConf, args);

	if (null == cmd) {
		return 0;
	}
	final String vertexClassName = args[0];
	final String jobName = "Giraph: " + vertexClassName;

	GiraphJob job = new GiraphJob(giraphConf, jobName);
	prepareHadoopMRJob(job, cmd);
	boolean verbose = !cmd.hasOption('q');

	return job.run(verbose) ? 0 : -1;
}
 

import org.apache.giraph.job.GiraphJob; //导入依赖的package包/类
@Override
public int run(String[] args) throws Exception {
	if (null == getConf()) {
		conf = new Configuration();
	}
	GiraphConfiguration giraphConf = new GiraphConfiguration(getConf());
	CommandLine cmd = ConfigurationUtils.parseArgs(giraphConf, args);

	if (null == cmd) {
		return 0;
	}
	final String vertexClassName = args[0];
	final String jobName = "Giraph: " + vertexClassName;

	giraphConf.setMaxNumberOfSupersteps(MAX_NUMBER_SUPERSTEPS);

	GiraphJob job = new GiraphJob(giraphConf, jobName);
	prepareHadoopMRJob(job, cmd);
	boolean verbose = !cmd.hasOption('q');

	return job.run(verbose) ? 0 : -1;
}
 

import org.apache.giraph.job.GiraphJob; //导入依赖的package包/类
@Override
public int run(String[] args) throws Exception {
  if (args.length != 2) {
    throw new IllegalArgumentException(
        "run: Must have 2 arguments <output path> <# of workers>");
  }
  GiraphJob job = new GiraphJob(getConf(), getClass().getName());
  job.getConfiguration().setComputationClass(SimpleComputation.class);
  job.getConfiguration().setVertexInputFormatClass(
      SimpleSuperstepVertexInputFormat.class);
  job.getConfiguration().setWorkerContextClass(EmitterWorkerContext.class);
  job.getConfiguration().set(
      SimpleVertexWithWorkerContext.OUTPUTDIR, args[0]);
  job.getConfiguration().setWorkerConfiguration(Integer.parseInt(args[1]),
      Integer.parseInt(args[1]),
      100.0f);
  if (job.run(true)) {
    return 0;
  } else {
    return -1;
  }
}
 

import org.apache.giraph.job.GiraphJob; //导入依赖的package包/类
/**
 * Run a sample BSP job in JobTracker, kill a task, and make sure
 * the job fails (not enough attempts to restart)
 *
 * @throws IOException
 * @throws ClassNotFoundException
 * @throws InterruptedException
 */
@Test
public void testBspFail()
    throws IOException, InterruptedException, ClassNotFoundException {
  // Allow this test only to be run on a real Hadoop setup
  if (!runningInDistributedMode()) {
    System.out.println("testBspFail: not executed for local setup.");
    return;
  }

  GiraphConfiguration conf = new GiraphConfiguration();
  conf.setComputationClass(SimpleFailComputation.class);
  conf.setVertexInputFormatClass(SimplePageRankVertexInputFormat.class);
  GiraphJob job = prepareJob(getCallingMethodName(), conf,
      getTempPath(getCallingMethodName()));
  job.getConfiguration().setInt("mapred.map.max.attempts", 1);
  assertTrue(!job.run(true));
}
 

import org.apache.giraph.job.GiraphJob; //导入依赖的package包/类
/**
 * Run a sample BSP job locally and test supersteps.
 *
 * @throws IOException
 * @throws ClassNotFoundException
 * @throws InterruptedException
 */
@Test
public void testBspSuperStep()
    throws IOException, InterruptedException, ClassNotFoundException {
  String callingMethod = getCallingMethodName();
  Path outputPath = getTempPath(callingMethod);
  GiraphConfiguration conf = new GiraphConfiguration();
  conf.setComputationClass(SimpleSuperstepComputation.class);
  conf.setVertexInputFormatClass(SimpleSuperstepVertexInputFormat.class);
  conf.setVertexOutputFormatClass(SimpleSuperstepVertexOutputFormat.class);
  GiraphJob job = prepareJob(callingMethod, conf, outputPath);
  Configuration configuration = job.getConfiguration();
  // GeneratedInputSplit will generate 10 vertices
  GeneratedVertexReader.READER_VERTICES.set(configuration, 10);
  assertTrue(job.run(true));
  if (!runningInDistributedMode()) {
    FileStatus fileStatus = getSinglePartFileStatus(configuration, outputPath);
    assertEquals(49l, fileStatus.getLen());
  }
}
 

import org.apache.giraph.job.GiraphJob; //导入依赖的package包/类
/**
 * Run a sample BSP job locally and test shortest paths.
 *
 * @throws IOException
 * @throws ClassNotFoundException
 * @throws InterruptedException
 */
@Test
public void testBspShortestPaths()
    throws IOException, InterruptedException, ClassNotFoundException {
  Path outputPath = getTempPath(getCallingMethodName());
  GiraphConfiguration conf = new GiraphConfiguration();
  conf.setComputationClass(SimpleShortestPathsComputation.class);
  conf.setVertexInputFormatClass(SimplePageRankVertexInputFormat.class);
  conf.setVertexOutputFormatClass(
      JsonLongDoubleFloatDoubleVertexOutputFormat.class);
  SimpleShortestPathsComputation.SOURCE_ID.set(conf, 0);
  GiraphJob job = prepareJob(getCallingMethodName(), conf, outputPath);

  assertTrue(job.run(true));

  int numResults = getNumResults(job.getConfiguration(), outputPath);

  int expectedNumResults = runningInDistributedMode() ? 15 : 5;
  assertEquals(expectedNumResults, numResults);
}
 

import org.apache.giraph.job.GiraphJob; //导入依赖的package包/类
/**
 * Run a sample BSP job locally and test MasterCompute.
 *
 * @throws IOException
 * @throws ClassNotFoundException
 * @throws InterruptedException
 */
@Test
public void testBspMasterCompute()
    throws IOException, InterruptedException, ClassNotFoundException {
  GiraphConfiguration conf = new GiraphConfiguration();
  conf.setComputationClass(SimpleMasterComputeComputation.class);
  conf.setVertexInputFormatClass(SimplePageRankVertexInputFormat.class);
  conf.setMasterComputeClass(
      SimpleMasterComputeComputation.SimpleMasterCompute.class);
  conf.setWorkerContextClass(
      SimpleMasterComputeComputation.SimpleMasterComputeWorkerContext.class);
  GiraphJob job = prepareJob(getCallingMethodName(), conf);
  assertTrue(job.run(true));
  if (!runningInDistributedMode()) {
    double finalSum =
        SimpleMasterComputeComputation.SimpleMasterComputeWorkerContext.getFinalSum();
    System.out.println("testBspMasterCompute: finalSum=" + finalSum);
    assertEquals(32.5, finalSum, 0d);
  }
}
 

import org.apache.giraph.job.GiraphJob; //导入依赖的package包/类
@Test
public void testComputationState() throws IOException,
    ClassNotFoundException, InterruptedException {
  if (runningInDistributedMode()) {
    System.out.println(
        "testComputeContext: Ignore this test in distributed mode.");
    return;
  }
  GiraphConfiguration conf = new GiraphConfiguration();
  conf.setComputationClass(TestComputationStateComputation.class);
  conf.setVertexInputFormatClass(
      SimplePageRankComputation.SimplePageRankVertexInputFormat.class);
  conf.setWorkerContextClass(
      TestComputationStateComputation.TestComputationStateWorkerContext.class);
  GiraphJob job = prepareJob(getCallingMethodName(), conf);
  // Use multithreading
  job.getConfiguration().setNumComputeThreads(
      TestComputationStateComputation.NUM_COMPUTE_THREADS);
  // Increase the number of vertices
  GeneratedVertexReader.READER_VERTICES.set(job.getConfiguration(),
      TestComputationStateComputation.NUM_VERTICES);
  // Increase the number of partitions
  GiraphConstants.USER_PARTITION_COUNT.set(job.getConfiguration(),
      TestComputationStateComputation.NUM_PARTITIONS);
  assertTrue(job.run(true));
}
 

import org.apache.giraph.job.GiraphJob; //导入依赖的package包/类
/** Tests if aggregators are handled on a proper way during supersteps */
@Test
public void testAggregatorsHandling() throws IOException,
    ClassNotFoundException, InterruptedException {
  GiraphConfiguration conf = new GiraphConfiguration();
  conf.setComputationClass(AggregatorsTestComputation.class);
  conf.setVertexInputFormatClass(
      AggregatorsTestComputation.SimpleVertexInputFormat.class);
  conf.setEdgeInputFormatClass(
      AggregatorsTestComputation.SimpleEdgeInputFormat.class);
  GiraphJob job = prepareJob(getCallingMethodName(), conf);
  job.getConfiguration().setMasterComputeClass(
      AggregatorsTestComputation.AggregatorsTestMasterCompute.class);
  // test with aggregators split in a few requests
  job.getConfiguration().setInt(
      AggregatorUtils.MAX_BYTES_PER_AGGREGATOR_REQUEST, 50);
  assertTrue(job.run(true));
}
 

import org.apache.giraph.job.GiraphJob; //导入依赖的package包/类
/**
 * Run a job that tests that this job completes in the desired number of
 * supersteps
 *
 * @throws java.io.IOException
 * @throws ClassNotFoundException
 * @throws InterruptedException
 */
@Test
public void testMaxSuperstep()
        throws IOException, InterruptedException, ClassNotFoundException {
  GiraphConfiguration conf = new GiraphConfiguration();
  conf.setComputationClass(InfiniteLoopComputation.class);
  conf.setVertexInputFormatClass(SimplePageRankVertexInputFormat.class);
  conf.setVertexOutputFormatClass(SimplePageRankVertexOutputFormat.class);
  GiraphJob job = prepareJob(getCallingMethodName(), conf,
      getTempPath(getCallingMethodName()));
  job.getConfiguration().setMaxNumberOfSupersteps(3);
  assertTrue(job.run(true));
  if (!runningInDistributedMode()) {
    GiraphHadoopCounter superstepCounter =
        GiraphStats.getInstance().getSuperstepCounter();
    assertEquals(superstepCounter.getValue(), 3L);
  }
}
 

import org.apache.giraph.job.GiraphJob; //导入依赖的package包/类
@Override public int run(String[] args) throws Exception {
  args = HiveJythonUtils.processArgs(args, CONF);
  LOG.info("Processed hive options now have args: " + Arrays.toString(args));

  HiveIO.init(CONF, false);

  PythonInterpreter interpreter = new PythonInterpreter();

  JythonJob jythonJob = parseJythonFiles(interpreter, args);

  logOptions();

  for (String arg : args) {
    Path remoteScriptPath = copyAndAdd(new Path(arg), CONF);
    ScriptLoader.addScriptToLoad(CONF, remoteScriptPath.toString(),
        DeployType.DISTRIBUTED_CACHE, Language.JYTHON);
  }

  String name = HiveJythonUtils.writeJythonJobToConf(jythonJob, CONF,
     interpreter);

  GiraphJob job = new GiraphJob(CONF, name);
  return job.run(true) ? 0 : -1;
}
 

import org.apache.giraph.job.GiraphJob; //导入依赖的package包/类
@Override
public int run(String[] args) throws Exception {
  if (args.length != 2) {
    throw new IllegalArgumentException(
        "run: Must have 2 arguments <output path> <# of workers>");
  }
  GiraphJob job = new GiraphJob(getConf(), getClass().getName());
  job.getConfiguration().setVertexClass(SimpleVertex.class);
  job.getConfiguration().setVertexInputFormatClass(
      SimpleSuperstepVertexInputFormat.class);
  job.getConfiguration().setWorkerContextClass(EmitterWorkerContext.class);
  job.getConfiguration().set(
      SimpleVertexWithWorkerContext.OUTPUTDIR, args[0]);
  job.getConfiguration().setWorkerConfiguration(Integer.parseInt(args[1]),
      Integer.parseInt(args[1]),
      100.0f);
  if (job.run(true)) {
    return 0;
  } else {
    return -1;
  }
}
 

import org.apache.giraph.job.GiraphJob; //导入依赖的package包/类
/**
 * Run a sample BSP job in JobTracker, kill a task, and make sure
 * the job fails (not enough attempts to restart)
 *
 * @throws IOException
 * @throws ClassNotFoundException
 * @throws InterruptedException
 */
@Test
public void testBspFail()
    throws IOException, InterruptedException, ClassNotFoundException {
  // Allow this test only to be run on a real Hadoop setup
  if (!runningInDistributedMode()) {
    System.out.println("testBspFail: not executed for local setup.");
    return;
  }

  GiraphConfiguration conf = new GiraphConfiguration();
  conf.setVertexClass(SimpleFailVertex.class);
  conf.setVertexInputFormatClass(SimplePageRankVertexInputFormat.class);
  GiraphJob job = prepareJob(getCallingMethodName(), conf,
      getTempPath(getCallingMethodName()));
  job.getConfiguration().setInt("mapred.map.max.attempts", 1);
  assertTrue(!job.run(true));
}
 

import org.apache.giraph.job.GiraphJob; //导入依赖的package包/类
/**
 * Run a sample BSP job locally and test supersteps.
 *
 * @throws IOException
 * @throws ClassNotFoundException
 * @throws InterruptedException
 */
@Test
public void testBspSuperStep()
    throws IOException, InterruptedException, ClassNotFoundException {
  String callingMethod = getCallingMethodName();
  Path outputPath = getTempPath(callingMethod);
  GiraphConfiguration conf = new GiraphConfiguration();
  conf.setVertexClass(SimpleSuperstepVertex.class);
  conf.setVertexInputFormatClass(SimpleSuperstepVertexInputFormat.class);
  conf.setVertexOutputFormatClass(SimpleSuperstepVertexOutputFormat.class);
  GiraphJob job = prepareJob(callingMethod, conf, outputPath);
  Configuration configuration = job.getConfiguration();
  // GeneratedInputSplit will generate 10 vertices
  configuration.setLong(GeneratedVertexReader.READER_VERTICES, 10);
  assertTrue(job.run(true));
  if (!runningInDistributedMode()) {
    FileStatus fileStatus = getSinglePartFileStatus(configuration, outputPath);
    assertEquals(49l, fileStatus.getLen());
  }
}