Java GiraphConfiguration.setWorkerContextClass方法代码示例

import org.apache.giraph.conf.GiraphConfiguration; //导入方法依赖的package包/类
/**
 * Run a sample BSP job locally and test MasterCompute.
 *
 * @throws IOException
 * @throws ClassNotFoundException
 * @throws InterruptedException
 */
@Test
public void testBspMasterCompute()
    throws IOException, InterruptedException, ClassNotFoundException {
  GiraphConfiguration conf = new GiraphConfiguration();
  conf.setComputationClass(SimpleMasterComputeComputation.class);
  conf.setVertexInputFormatClass(SimplePageRankVertexInputFormat.class);
  conf.setMasterComputeClass(
      SimpleMasterComputeComputation.SimpleMasterCompute.class);
  conf.setWorkerContextClass(
      SimpleMasterComputeComputation.SimpleMasterComputeWorkerContext.class);
  GiraphJob job = prepareJob(getCallingMethodName(), conf);
  assertTrue(job.run(true));
  if (!runningInDistributedMode()) {
    double finalSum =
        SimpleMasterComputeComputation.SimpleMasterComputeWorkerContext.getFinalSum();
    System.out.println("testBspMasterCompute: finalSum=" + finalSum);
    assertEquals(32.5, finalSum, 0d);
  }
}
 

import org.apache.giraph.conf.GiraphConfiguration; //导入方法依赖的package包/类
/**
 * Run a sample BSP job locally and test MasterCompute.
 *
 * @throws IOException
 * @throws ClassNotFoundException
 * @throws InterruptedException
 */
@Test
public void testBspMasterCompute()
    throws IOException, InterruptedException, ClassNotFoundException {
  GiraphConfiguration conf = new GiraphConfiguration();
  conf.setVertexClass(SimpleMasterComputeVertex.class);
  conf.setVertexInputFormatClass(SimplePageRankVertexInputFormat.class);
  conf.setMasterComputeClass(
      SimpleMasterComputeVertex.SimpleMasterCompute.class);
  conf.setWorkerContextClass(
      SimpleMasterComputeVertex.SimpleMasterComputeWorkerContext.class);
  GiraphJob job = prepareJob(getCallingMethodName(), conf);
  assertTrue(job.run(true));
  if (!runningInDistributedMode()) {
    double finalSum =
        SimpleMasterComputeVertex.SimpleMasterComputeWorkerContext.getFinalSum();
    System.out.println("testBspMasterCompute: finalSum=" + finalSum);
    assertEquals(32.5, finalSum, 0d);
  }
}
 

import org.apache.giraph.conf.GiraphConfiguration; //导入方法依赖的package包/类
@Override
protected void prepareConfiguration(GiraphConfiguration conf,
    CommandLine cmd) {
  conf.setVertexClass(RandomMessageVertex.class);
  conf.setVertexInputFormatClass(PseudoRandomVertexInputFormat.class);
  conf.setWorkerContextClass(RandomMessageBenchmarkWorkerContext.class);
  conf.setMasterComputeClass(RandomMessageBenchmarkMasterCompute.class);
  conf.setLong(PseudoRandomInputFormatConstants.AGGREGATE_VERTICES,
      BenchmarkOption.VERTICES.getOptionLongValue(cmd));
  conf.setLong(PseudoRandomInputFormatConstants.EDGES_PER_VERTEX,
      BenchmarkOption.EDGES_PER_VERTEX.getOptionLongValue(cmd));
  conf.setInt(SUPERSTEP_COUNT,
      BenchmarkOption.SUPERSTEPS.getOptionIntValue(cmd));
  conf.setInt(RandomMessageBenchmark.NUM_BYTES_PER_MESSAGE,
      BYTES_PER_MESSAGE.getOptionIntValue(cmd));
  conf.setInt(RandomMessageBenchmark.NUM_MESSAGES_PER_EDGE,
      MESSAGES_PER_EDGE.getOptionIntValue(cmd));
  if (FLUSH_THREADS.optionTurnedOn(cmd)) {
    conf.setInt(GiraphConstants.MSG_NUM_FLUSH_THREADS,
        FLUSH_THREADS.getOptionIntValue(cmd));
  }
}
 

import org.apache.giraph.conf.GiraphConfiguration; //导入方法依赖的package包/类
@Override
public int run(String[] args) throws Exception {
    if (args.length != 3) {
        throw new IllegalArgumentException(
            "Syntax error: Must have 3 arguments <numbersOfWorkers> <inputLocaiton> <outputLocation>");
    }

    int numberOfWorkers = Integer.parseInt(args[0]);
    String inputLocation = args[1];
    String outputLocation = args[2];

    GiraphJob job = new GiraphJob(getConf(), getClass().getName());
    GiraphConfiguration gconf = job.getConfiguration();
    gconf.setWorkerConfiguration(numberOfWorkers, numberOfWorkers, 100.0f);

    GiraphFileInputFormat.addVertexInputPath(gconf, new Path(inputLocation));
    FileOutputFormat.setOutputPath(job.getInternalJob(), new Path(outputLocation));

    gconf.setComputationClass(ZombieComputation.class);
    gconf.setMasterComputeClass(ZombieMasterCompute.class);
    gconf.setVertexInputFormatClass(ZombieTextVertexInputFormat.class);
    gconf.setVertexOutputFormatClass(ZombieTextVertexOutputFormat.class);
    gconf.setWorkerContextClass(ZombieWorkerContext.class);

    boolean verbose = true;
    if (job.run(verbose)) {
        return 0;
    } else {
        return -1;
    }
}
 

import org.apache.giraph.conf.GiraphConfiguration; //导入方法依赖的package包/类
/**
 * A local integration test on toy data
 */
@Test
public void testWeightedGraph() throws Exception {
  // A small graph
  String[] graph =
      new String[] { "12 34:0.1 56:0.9", "34 78:0.9 56:0.1",
        "56 12:0.1 34:0.8 78:0.1", "78 34:1.0" };

  GiraphConfiguration conf = new GiraphConfiguration();
  conf.setInt(RandomWalkWithRestartVertex.SOURCE_VERTEX, 12);
  conf.setInt(RandomWalkWithRestartVertex.MAX_SUPERSTEPS, 30);
  conf.setFloat(RandomWalkWithRestartVertex.TELEPORTATION_PROBABILITY, 0.15f);
  conf.setVertexClass(RandomWalkWithRestartVertex.class);
  conf.setOutEdgesClass(ByteArrayEdges.class);
  conf.setVertexInputFormatClass(
      NormalizingLongDoubleDoubleTextInputFormat.class);
  conf.setVertexOutputFormatClass(
      VertexWithDoubleValueDoubleEdgeTextOutputFormat.class);
  conf.setWorkerContextClass(RandomWalkWorkerContext.class);
  conf.setMasterComputeClass(RandomWalkVertexMasterCompute.class);
  // Run internally
  Iterable<String> results = InternalVertexRunner.run(conf, graph);

  Map<Long, Double> steadyStateProbabilities =
      RandomWalkTestUtils.parseSteadyStateProbabilities(results);
  // values computed with external software
  // 0.163365, 0.378932, 0.156886, 0.300816
  assertEquals(0.163365, steadyStateProbabilities.get(12L),
      RandomWalkTestUtils.EPSILON);
  assertEquals(0.378932, steadyStateProbabilities.get(34L),
      RandomWalkTestUtils.EPSILON);
  assertEquals(0.156886, steadyStateProbabilities.get(56L),
      RandomWalkTestUtils.EPSILON);
  assertEquals(0.300816, steadyStateProbabilities.get(78L),
      RandomWalkTestUtils.EPSILON);
}
 

import org.apache.giraph.conf.GiraphConfiguration; //导入方法依赖的package包/类
/**
 * Generic page rank test
 *
 * @param numComputeThreads Number of compute threads to use
 * @throws java.io.IOException
 * @throws ClassNotFoundException
 * @throws InterruptedException
 */
private void testPageRank(int numComputeThreads)
    throws IOException, InterruptedException, ClassNotFoundException {
  GiraphConfiguration conf = new GiraphConfiguration();
  conf.setComputationClass(SimplePageRankComputation.class);
  conf.setVertexInputFormatClass(
      SimplePageRankComputation.SimplePageRankVertexInputFormat.class);
  conf.setWorkerContextClass(
      SimplePageRankComputation.SimplePageRankWorkerContext.class);
  conf.setMasterComputeClass(
      SimplePageRankComputation.SimplePageRankMasterCompute.class);
  conf.setNumComputeThreads(numComputeThreads);
  // Set enough partitions to generate randomness on the compute side
  if (numComputeThreads != 1) {
    GiraphConstants.USER_PARTITION_COUNT.set(conf, numComputeThreads * 5);
  }
  GiraphJob job = prepareJob(getCallingMethodName(), conf);
  assertTrue(job.run(true));
  if (!runningInDistributedMode()) {
    double maxPageRank =
        SimplePageRankComputation.SimplePageRankWorkerContext.getFinalMax();
    double minPageRank =
        SimplePageRankComputation.SimplePageRankWorkerContext.getFinalMin();
    long numVertices =
        SimplePageRankComputation.SimplePageRankWorkerContext.getFinalSum();
    System.out.println(getCallingMethodName() + ": maxPageRank=" +
        maxPageRank + " minPageRank=" +
        minPageRank + " numVertices=" + numVertices + ", " +
        " numComputeThreads=" + numComputeThreads);
    assertEquals(34.03, maxPageRank, 0.001);
    assertEquals(0.03, minPageRank, 0.00001);
    assertEquals(5L, numVertices);
  }
}
 

import org.apache.giraph.conf.GiraphConfiguration; //导入方法依赖的package包/类
/**
 * A local integration test on toy data
 */
@Test
public void testToyData() throws Exception {
  // A small graph
  String[] graph = new String[] { "12 34 56", "34 78", "56 34 78", "78 34" };

  GiraphConfiguration conf = new GiraphConfiguration();
  conf.setInt(RandomWalkWithRestartComputation.SOURCE_VERTEX, 12);
  conf.setInt(RandomWalkWithRestartComputation.MAX_SUPERSTEPS, 30);
  conf.setFloat(
      RandomWalkWithRestartComputation.TELEPORTATION_PROBABILITY, 0.25f);
  conf.setComputationClass(RandomWalkWithRestartComputation.class);
  conf.setOutEdgesClass(ByteArrayEdges.class);
  conf.setVertexInputFormatClass(LongDoubleDoubleTextInputFormat.class);
  conf.setVertexOutputFormatClass(
      VertexWithDoubleValueDoubleEdgeTextOutputFormat.class);
  conf.setWorkerContextClass(RandomWalkWorkerContext.class);
  conf.setMasterComputeClass(RandomWalkVertexMasterCompute.class);
  // Run internally
  Iterable<String> results = InternalVertexRunner.run(conf, graph);

  Map<Long, Double> steadyStateProbabilities =
      RandomWalkTestUtils.parseSteadyStateProbabilities(results);
  // values computed with external software
  // 0.25, 0.354872, 0.09375, 0.301377
  assertEquals(0.25, steadyStateProbabilities.get(12L), RandomWalkTestUtils.EPSILON);
  assertEquals(0.354872, steadyStateProbabilities.get(34L),
      RandomWalkTestUtils.EPSILON);
  assertEquals(0.09375, steadyStateProbabilities.get(56L), RandomWalkTestUtils.EPSILON);
  assertEquals(0.301377, steadyStateProbabilities.get(78L),
      RandomWalkTestUtils.EPSILON);
}
 

import org.apache.giraph.conf.GiraphConfiguration; //导入方法依赖的package包/类
/**
 * Run a job that requires checkpointing and will have a worker crash
 * and still recover from a previous checkpoint.
 *
 * @throws IOException
 * @throws ClassNotFoundException
 * @throws InterruptedException
 */
@Test
public void testSingleFault()
  throws IOException, InterruptedException, ClassNotFoundException {
  if (!runningInDistributedMode()) {
    System.out.println(
        "testSingleFault: Ignore this test in local mode.");
    return;
  }
  Path outputPath = getTempPath(getCallingMethodName());
  GiraphConfiguration conf = new GiraphConfiguration();
  conf.setVertexClass(
      SimpleCheckpointVertex.SimpleCheckpointComputation.class);
  conf.setWorkerContextClass(
      SimpleCheckpointVertex.SimpleCheckpointVertexWorkerContext.class);
  conf.setMasterComputeClass(
      SimpleCheckpointVertex.SimpleCheckpointVertexMasterCompute.class);
  conf.setVertexInputFormatClass(SimpleSuperstepVertexInputFormat.class);
  conf.setVertexOutputFormatClass(SimpleSuperstepVertexOutputFormat.class);
  conf.setBoolean(SimpleCheckpointVertex.ENABLE_FAULT, true);
  conf.setInt("mapred.map.max.attempts", 4);
  // Trigger failure faster
  conf.setInt("mapred.task.timeout", 10000);
  conf.setMaxMasterSuperstepWaitMsecs(10000);
  conf.setEventWaitMsecs(1000);
  conf.setCheckpointFrequency(2);
  GiraphConstants.CHECKPOINT_DIRECTORY.set(conf,
      getTempPath("_singleFaultCheckpoints").toString());
  GiraphConstants.CLEANUP_CHECKPOINTS_AFTER_SUCCESS.set(conf, false);
  GiraphConstants.ZOOKEEPER_SESSION_TIMEOUT.set(conf, 10000);
  GiraphConstants.ZOOKEEPER_MIN_SESSION_TIMEOUT.set(conf, 10000);
  GiraphJob job = prepareJob(getCallingMethodName(), conf, outputPath);
  assertTrue(job.run(true));
}
 

import org.apache.giraph.conf.GiraphConfiguration; //导入方法依赖的package包/类
/**
 * Run a job that tests the various graph mutations that can occur
 *
 * @throws IOException
 * @throws ClassNotFoundException
 * @throws InterruptedException
 */
@Test
public void testMutateGraph()
        throws IOException, InterruptedException, ClassNotFoundException {
  GiraphConfiguration conf = new GiraphConfiguration();
  conf.setComputationClass(SimpleMutateGraphComputation.class);
  conf.setVertexInputFormatClass(SimplePageRankVertexInputFormat.class);
  conf.setVertexOutputFormatClass(SimplePageRankVertexOutputFormat.class);
  conf.setWorkerContextClass(
      SimpleMutateGraphComputation.SimpleMutateGraphVertexWorkerContext.class);
  GiraphJob job = prepareJob(getCallingMethodName(), conf,
      getTempPath(getCallingMethodName()));
  assertTrue(job.run(true));
}
 

import org.apache.giraph.conf.GiraphConfiguration; //导入方法依赖的package包/类
/**
 * A local integration test on toy data
 */
@Test
public void testToyData() throws Exception {
  // A small graph
  String[] graph = new String[] { "12 34 56", "34 78", "56 34 78", "78 34" };

  GiraphConfiguration conf = new GiraphConfiguration();
  conf.setInt(RandomWalkWithRestartVertex.SOURCE_VERTEX, 12);
  conf.setInt(RandomWalkWithRestartVertex.MAX_SUPERSTEPS, 30);
  conf.setFloat(RandomWalkWithRestartVertex.TELEPORTATION_PROBABILITY, 0.25f);
  conf.setVertexClass(RandomWalkWithRestartVertex.class);
  conf.setOutEdgesClass(ByteArrayEdges.class);
  conf.setVertexInputFormatClass(LongDoubleDoubleTextInputFormat.class);
  conf.setVertexOutputFormatClass(
      VertexWithDoubleValueDoubleEdgeTextOutputFormat.class);
  conf.setWorkerContextClass(RandomWalkWorkerContext.class);
  conf.setMasterComputeClass(RandomWalkVertexMasterCompute.class);
  // Run internally
  Iterable<String> results = InternalVertexRunner.run(conf, graph);

  Map<Long, Double> steadyStateProbabilities =
      RandomWalkTestUtils.parseSteadyStateProbabilities(results);
  // values computed with external software
  // 0.25, 0.354872, 0.09375, 0.301377
  assertEquals(0.25, steadyStateProbabilities.get(12L), RandomWalkTestUtils.EPSILON);
  assertEquals(0.354872, steadyStateProbabilities.get(34L),
      RandomWalkTestUtils.EPSILON);
  assertEquals(0.09375, steadyStateProbabilities.get(56L), RandomWalkTestUtils.EPSILON);
  assertEquals(0.301377, steadyStateProbabilities.get(78L),
      RandomWalkTestUtils.EPSILON);
}
 

import org.apache.giraph.conf.GiraphConfiguration; //导入方法依赖的package包/类
/**
 * Run a job that requires checkpointing and will have a worker crash
 * and still recover from a previous checkpoint.
 *
 * @throws IOException
 * @throws ClassNotFoundException
 * @throws InterruptedException
 */
@Test
public void testSingleFault()
  throws IOException, InterruptedException, ClassNotFoundException {
  if (!runningInDistributedMode()) {
    System.out.println(
        "testSingleFault: Ignore this test in local mode.");
    return;
  }
  Path outputPath = getTempPath(getCallingMethodName());
  GiraphConfiguration conf = new GiraphConfiguration();
  conf.setComputationClass(
      SimpleCheckpoint.SimpleCheckpointComputation.class);
  conf.setWorkerContextClass(
      SimpleCheckpoint.SimpleCheckpointVertexWorkerContext.class);
  conf.setMasterComputeClass(
      SimpleCheckpoint.SimpleCheckpointVertexMasterCompute.class);
  conf.setVertexInputFormatClass(SimpleSuperstepVertexInputFormat.class);
  conf.setVertexOutputFormatClass(SimpleSuperstepVertexOutputFormat.class);
  conf.setBoolean(SimpleCheckpoint.ENABLE_FAULT, true);
  conf.setInt("mapred.map.max.attempts", 4);
  // Trigger failure faster
  conf.setInt("mapred.task.timeout", 10000);
  conf.setMaxMasterSuperstepWaitMsecs(10000);
  conf.setEventWaitMsecs(1000);
  conf.setCheckpointFrequency(2);
  GiraphConstants.CHECKPOINT_DIRECTORY.set(conf,
      getTempPath("_singleFaultCheckpoints").toString());
  GiraphConstants.CLEANUP_CHECKPOINTS_AFTER_SUCCESS.set(conf, false);
  GiraphConstants.ZOOKEEPER_SESSION_TIMEOUT.set(conf, 10000);
  GiraphConstants.ZOOKEEPER_MIN_SESSION_TIMEOUT.set(conf, 10000);
  GiraphJob job = prepareJob(getCallingMethodName(), conf, outputPath);
  assertTrue(job.run(true));
}
 

import org.apache.giraph.conf.GiraphConfiguration; //导入方法依赖的package包/类
@Override
protected void prepareConfiguration(GiraphConfiguration conf,
    CommandLine cmd) {
  conf.setVertexClass(AggregatorsBenchmarkVertex.class);
  conf.setMasterComputeClass(AggregatorsBenchmarkMasterCompute.class);
  conf.setVertexInputFormatClass(PseudoRandomVertexInputFormat.class);
  conf.setWorkerContextClass(AggregatorsBenchmarkWorkerContext.class);
  conf.setLong(PseudoRandomInputFormatConstants.AGGREGATE_VERTICES,
      BenchmarkOption.VERTICES.getOptionLongValue(cmd));
  conf.setLong(PseudoRandomInputFormatConstants.EDGES_PER_VERTEX, 1);
  conf.setInt(AGGREGATORS_NUM, AGGREGATORS.getOptionIntValue(cmd));
  conf.setInt("workers", conf.getInt(GiraphConstants.MAX_WORKERS, -1));
}
 

import org.apache.giraph.conf.GiraphConfiguration; //导入方法依赖的package包/类
/**
 * Run a job that tests the various graph mutations that can occur
 *
 * @throws IOException
 * @throws ClassNotFoundException
 * @throws InterruptedException
 */
@Test
public void testMutateGraph()
        throws IOException, InterruptedException, ClassNotFoundException {
  GiraphConfiguration conf = new GiraphConfiguration();
  conf.setVertexClass(SimpleMutateGraphVertex.class);
  conf.setVertexInputFormatClass(SimplePageRankVertexInputFormat.class);
  conf.setVertexOutputFormatClass(SimplePageRankVertexOutputFormat.class);
  conf.setWorkerContextClass(
      SimpleMutateGraphVertex.SimpleMutateGraphVertexWorkerContext.class);
  GiraphJob job = prepareJob(getCallingMethodName(), conf,
      getTempPath(getCallingMethodName()));
  assertTrue(job.run(true));
}
 

import org.apache.giraph.conf.GiraphConfiguration; //导入方法依赖的package包/类
/**
 * Generic page rank test
 *
 * @param numComputeThreads Number of compute threads to use
 * @throws java.io.IOException
 * @throws ClassNotFoundException
 * @throws InterruptedException
 */
private void testPageRank(int numComputeThreads)
    throws IOException, InterruptedException, ClassNotFoundException {
  GiraphConfiguration conf = new GiraphConfiguration();
  conf.setVertexClass(SimplePageRankVertex.class);
  conf.setVertexInputFormatClass(
      SimplePageRankVertex.SimplePageRankVertexInputFormat.class);
  conf.setWorkerContextClass(
      SimplePageRankVertex.SimplePageRankVertexWorkerContext.class);
  conf.setMasterComputeClass(
      SimplePageRankVertex.SimplePageRankVertexMasterCompute.class);
  conf.setNumComputeThreads(numComputeThreads);
  // Set enough partitions to generate randomness on the compute side
  if (numComputeThreads != 1) {
    GiraphConstants.USER_PARTITION_COUNT.set(conf, numComputeThreads * 5);
  }
  GiraphJob job = prepareJob(getCallingMethodName(), conf);
  assertTrue(job.run(true));
  if (!runningInDistributedMode()) {
    double maxPageRank =
        SimplePageRankVertex.SimplePageRankVertexWorkerContext.getFinalMax();
    double minPageRank =
        SimplePageRankVertex.SimplePageRankVertexWorkerContext.getFinalMin();
    long numVertices =
        SimplePageRankVertex.SimplePageRankVertexWorkerContext.getFinalSum();
    System.out.println(getCallingMethodName() + ": maxPageRank=" +
        maxPageRank + " minPageRank=" +
        minPageRank + " numVertices=" + numVertices + ", " +
        " numComputeThreads=" + numComputeThreads);
    assertEquals(34.03, maxPageRank, 0.001);
    assertEquals(0.03, minPageRank, 0.00001);
    assertEquals(5L, numVertices);
  }
}
 

import org.apache.giraph.conf.GiraphConfiguration; //导入方法依赖的package包/类
@Test
public void testPartitionContext() throws IOException,
    ClassNotFoundException, InterruptedException {
  if (runningInDistributedMode()) {
    System.out.println(
        "testComputeContext: Ignore this test in distributed mode.");
    return;
  }
  GiraphConfiguration conf = new GiraphConfiguration();
  conf.setVertexClass(PartitionContextTestVertex.class);
  conf.setVertexInputFormatClass(
      SimplePageRankVertex.SimplePageRankVertexInputFormat.class);
  conf.setWorkerContextClass(
      PartitionContextTestVertex.TestPartitionContextWorkerContext.class);
  conf.setPartitionContextClass(
      PartitionContextTestVertex.TestPartitionContextPartitionContext.class);
  GiraphJob job = prepareJob(getCallingMethodName(), conf);
  // Use multithreading
  job.getConfiguration().setNumComputeThreads(
      PartitionContextTestVertex.NUM_COMPUTE_THREADS);
  // Increase the number of vertices
  job.getConfiguration().setInt(
      GeneratedVertexReader.READER_VERTICES,
      PartitionContextTestVertex.NUM_VERTICES);
  // Increase the number of partitions
  GiraphConstants.USER_PARTITION_COUNT.set(job.getConfiguration(),
      PartitionContextTestVertex.NUM_PARTITIONS);
  assertTrue(job.run(true));
}