Java GiraphConfiguration.setVertexClass方法代码示例

import org.apache.giraph.conf.GiraphConfiguration; //导入方法依赖的package包/类
@Before
public void prepare() throws IOException {
  directory = Files.createTempDir();

  Configuration.addDefaultResource("giraph-site.xml");
  GiraphConfiguration initConfig = new GiraphConfiguration();
  initConfig.setVertexClass(IntVertex.class);
  GiraphConstants.MESSAGES_DIRECTORY.set(
      initConfig, new File(directory, "giraph_messages").toString());
  config = new ImmutableClassesGiraphConfiguration(initConfig);

  testData = new TestData();
  testData.maxId = 1000000;
  testData.maxMessage = 1000000;
  testData.maxNumberOfMessages = 100;
  testData.numVertices = 50;
  testData.numTimes = 10;
  testData.numOfPartitions = 5;
  testData.maxMessagesInMemory = 20;

  service =
      MockUtils.mockServiceGetVertexPartitionOwner(testData.numOfPartitions);
}
 

import org.apache.giraph.conf.GiraphConfiguration; //导入方法依赖的package包/类
@Test
public void testGetsCalled() throws Exception {
  assertEquals(0, Obs.postApp);

  String[] graph = new String[] { "1", "2", "3" };

  String klasses[] = new String[] {
      Obs.class.getName(),
      Obs.class.getName()
  };

  GiraphConfiguration conf = new GiraphConfiguration();
  conf.set(GiraphConstants.MASTER_OBSERVER_CLASSES.getKey(),
      arrayToString(klasses));
  conf.setVertexClass(NoOpVertex.class);
  conf.setOutEdgesClass(ByteArrayEdges.class);
  conf.setVertexInputFormatClass(IntNullNullTextInputFormat.class);
  InternalVertexRunner.run(conf, graph);

  assertEquals(2, Obs.preApp);
  // 3 supersteps + 1 input superstep * 2 observers = 8 callbacks
  assertEquals(8, Obs.preSuperstep);
  assertEquals(8, Obs.postSuperstep);
  assertEquals(2, Obs.postApp);
}
 

import org.apache.giraph.conf.GiraphConfiguration; //导入方法依赖的package包/类
@Override
protected void prepareConfiguration(GiraphConfiguration conf,
    CommandLine cmd) {
  conf.setVertexClass(RandomMessageVertex.class);
  conf.setVertexInputFormatClass(PseudoRandomVertexInputFormat.class);
  conf.setWorkerContextClass(RandomMessageBenchmarkWorkerContext.class);
  conf.setMasterComputeClass(RandomMessageBenchmarkMasterCompute.class);
  conf.setLong(PseudoRandomInputFormatConstants.AGGREGATE_VERTICES,
      BenchmarkOption.VERTICES.getOptionLongValue(cmd));
  conf.setLong(PseudoRandomInputFormatConstants.EDGES_PER_VERTEX,
      BenchmarkOption.EDGES_PER_VERTEX.getOptionLongValue(cmd));
  conf.setInt(SUPERSTEP_COUNT,
      BenchmarkOption.SUPERSTEPS.getOptionIntValue(cmd));
  conf.setInt(RandomMessageBenchmark.NUM_BYTES_PER_MESSAGE,
      BYTES_PER_MESSAGE.getOptionIntValue(cmd));
  conf.setInt(RandomMessageBenchmark.NUM_MESSAGES_PER_EDGE,
      MESSAGES_PER_EDGE.getOptionIntValue(cmd));
  if (FLUSH_THREADS.optionTurnedOn(cmd)) {
    conf.setInt(GiraphConstants.MSG_NUM_FLUSH_THREADS,
        FLUSH_THREADS.getOptionIntValue(cmd));
  }
}
 

import org.apache.giraph.conf.GiraphConfiguration; //导入方法依赖的package包/类
/**
 * Run a job that tests that this job completes in the desired number of
 * supersteps
 *
 * @throws java.io.IOException
 * @throws ClassNotFoundException
 * @throws InterruptedException
 */
@Test
public void testMaxSuperstep()
        throws IOException, InterruptedException, ClassNotFoundException {
  GiraphConfiguration conf = new GiraphConfiguration();
  conf.setVertexClass(InfiniteLoopVertex.class);
  conf.setVertexInputFormatClass(SimplePageRankVertexInputFormat.class);
  conf.setVertexOutputFormatClass(SimplePageRankVertexOutputFormat.class);
  GiraphJob job = prepareJob(getCallingMethodName(), conf,
      getTempPath(getCallingMethodName()));
  job.getConfiguration().setMaxNumberOfSupersteps(3);
  assertTrue(job.run(true));
  if (!runningInDistributedMode()) {
    GiraphHadoopCounter superstepCounter =
        GiraphStats.getInstance().getSuperstepCounter();
    assertEquals(superstepCounter.getValue(), 3L);
  }
}
 

import org.apache.giraph.conf.GiraphConfiguration; //导入方法依赖的package包/类
/**
 * Test whether vertices with NullWritable for vertex value type, edge value
 * type and message value type can be instantiated.
 */
@Test
public void testInstantiateNullVertex() throws IOException {
  GiraphConfiguration nullConf = new GiraphConfiguration();
  nullConf.setVertexClass(NullVertex.class);
  ImmutableClassesGiraphConfiguration<
      NullWritable, NullWritable, NullWritable,
      NullWritable> immutableClassesGiraphConfiguration =
      new ImmutableClassesGiraphConfiguration<
          NullWritable, NullWritable, NullWritable, NullWritable>(
          nullConf);
  NullWritable vertexValue =
      immutableClassesGiraphConfiguration.createVertexValue();
  NullWritable edgeValue =
      immutableClassesGiraphConfiguration.createEdgeValue();
  NullWritable messageValue =
      immutableClassesGiraphConfiguration.createMessageValue();
  assertSame(vertexValue.getClass(), NullWritable.class);
  assertSame(vertexValue, edgeValue);
  assertSame(edgeValue, messageValue);
}
 

import org.apache.giraph.conf.GiraphConfiguration; //导入方法依赖的package包/类
@Override
protected void prepareConfiguration(GiraphConfiguration conf,
    CommandLine cmd) {
  conf.setVertexClass(PageRankVertex.class);
  conf.setOutEdgesClass(IntNullArrayEdges.class);
  conf.setCombinerClass(FloatSumCombiner.class);
  conf.setVertexInputFormatClass(
      PseudoRandomIntNullVertexInputFormat.class);

  conf.setInt(PseudoRandomInputFormatConstants.AGGREGATE_VERTICES,
      BenchmarkOption.VERTICES.getOptionIntValue(cmd));
  conf.setInt(PseudoRandomInputFormatConstants.EDGES_PER_VERTEX,
      BenchmarkOption.EDGES_PER_VERTEX.getOptionIntValue(cmd));
  conf.setInt(PageRankVertex.SUPERSTEP_COUNT,
      BenchmarkOption.SUPERSTEPS.getOptionIntValue(cmd));
  conf.setFloat(PseudoRandomInputFormatConstants.LOCAL_EDGES_MIN_RATIO,
      BenchmarkOption.LOCAL_EDGES_MIN_RATIO.getOptionFloatValue(cmd,
          PseudoRandomInputFormatConstants.LOCAL_EDGES_MIN_RATIO_DEFAULT));
}
 

import org.apache.giraph.conf.GiraphConfiguration; //导入方法依赖的package包/类
/**
 * Run a sample BSP job locally and test shortest paths.
 *
 * @throws IOException
 * @throws ClassNotFoundException
 * @throws InterruptedException
 */
@Test
public void testBspShortestPaths()
    throws IOException, InterruptedException, ClassNotFoundException {
  Path outputPath = getTempPath(getCallingMethodName());
  GiraphConfiguration conf = new GiraphConfiguration();
  conf.setVertexClass(SimpleShortestPathsVertex.class);
  conf.setVertexInputFormatClass(SimplePageRankVertexInputFormat.class);
  conf.setVertexOutputFormatClass(
      JsonLongDoubleFloatDoubleVertexOutputFormat.class);
  SimpleShortestPathsVertex.SOURCE_ID.set(conf, 0);
  GiraphJob job = prepareJob(getCallingMethodName(), conf, outputPath);

  assertTrue(job.run(true));

  int numResults = getNumResults(job.getConfiguration(), outputPath);

  int expectedNumResults = runningInDistributedMode() ? 15 : 5;
  assertEquals(expectedNumResults, numResults);
}
 

import org.apache.giraph.conf.GiraphConfiguration; //导入方法依赖的package包/类
@Test
public void testEdgesOnlyWithReverse() throws Exception {
  String[] edges = new String[] {
      "1 2",
      "2 3",
      "2 4",
      "4 1"
  };

  GiraphConfiguration conf = new GiraphConfiguration();
  conf.setVertexClass(TestVertexWithNumEdges.class);
  conf.setOutEdgesClass(ByteArrayEdges.class);
  conf.setEdgeInputFormatClass(IntNullReverseTextEdgeInputFormat.class);
  conf.setVertexOutputFormatClass(IdWithValueTextOutputFormat.class);
  Iterable<String> results = InternalVertexRunner.run(conf, null, edges);

  Map<Integer, Integer> values = parseResults(results);

  // Check that all vertices with outgoing edges have been created
  assertEquals(4, values.size());
  // Check the number of edges for each vertex
  assertEquals(2, (int) values.get(1));
  assertEquals(3, (int) values.get(2));
  assertEquals(1, (int) values.get(3));
  assertEquals(2, (int) values.get(4));
}
 

import org.apache.giraph.conf.GiraphConfiguration; //导入方法依赖的package包/类
/**
 * A local integration test on toy data
 */
@Test
public void testToyData() throws Exception {

  // A small graph
  String[] graph = new String[] {
    "1 4 2 3",
    "2 1",
    "4 3 2",
    "5 2 4"
  };

  GiraphConfiguration conf = new GiraphConfiguration();
  conf.setInt(RandomWalkWithRestartVertex.MAX_SUPERSTEPS, 50);
  conf.setFloat(RandomWalkWithRestartVertex.TELEPORTATION_PROBABILITY, 0.15f);
  conf.setVertexClass(PageRankVertex.class);
  conf.setOutEdgesClass(ByteArrayEdges.class);
  conf.setVertexInputFormatClass(LongDoubleNullTextInputFormat.class);
  conf.setVertexOutputFormatClass(
      VertexWithDoubleValueNullEdgeTextOutputFormat.class);
  conf.setWorkerContextClass(RandomWalkWorkerContext.class);
  conf.setMasterComputeClass(RandomWalkVertexMasterCompute.class);
  // Run internally
  Iterable<String> results = InternalVertexRunner.run(conf, graph);

  Map<Long, Double> steadyStateProbabilities =
      RandomWalkTestUtils.parseSteadyStateProbabilities(results);

  assertEquals(0.28159076008518047, steadyStateProbabilities.get(1l),
      RandomWalkTestUtils.EPSILON);
  assertEquals(0.2514648601529863, steadyStateProbabilities.get(2l),
      RandomWalkTestUtils.EPSILON);
  assertEquals(0.22262961972286327, steadyStateProbabilities.get(3l),
      RandomWalkTestUtils.EPSILON);
  assertEquals(0.17646783276703806, steadyStateProbabilities.get(4l),
      RandomWalkTestUtils.EPSILON);
  assertEquals(0.06784692727193153, steadyStateProbabilities.get(5l),
      RandomWalkTestUtils.EPSILON);
}
 

import org.apache.giraph.conf.GiraphConfiguration; //导入方法依赖的package包/类
@Before
public void setUp() {
  GiraphConfiguration giraphConfiguration = new GiraphConfiguration();
  giraphConfiguration.setVertexClass(DummyVertex.class);
  conf = new ImmutableClassesGiraphConfiguration<Text,
      DoubleWritable, DoubleWritable, Writable>(giraphConfiguration);
}
 

import org.apache.giraph.conf.GiraphConfiguration; //导入方法依赖的package包/类
/**
 * A local integration test on toy data
 */
@Test
public void testToyData() throws Exception {
  // A small graph
  String[] graph = new String[] { "12 34 56", "34 78", "56 34 78", "78 34" };

  GiraphConfiguration conf = new GiraphConfiguration();
  conf.setInt(RandomWalkWithRestartVertex.SOURCE_VERTEX, 12);
  conf.setInt(RandomWalkWithRestartVertex.MAX_SUPERSTEPS, 30);
  conf.setFloat(RandomWalkWithRestartVertex.TELEPORTATION_PROBABILITY, 0.25f);
  conf.setVertexClass(RandomWalkWithRestartVertex.class);
  conf.setOutEdgesClass(ByteArrayEdges.class);
  conf.setVertexInputFormatClass(LongDoubleDoubleTextInputFormat.class);
  conf.setVertexOutputFormatClass(
      VertexWithDoubleValueDoubleEdgeTextOutputFormat.class);
  conf.setWorkerContextClass(RandomWalkWorkerContext.class);
  conf.setMasterComputeClass(RandomWalkVertexMasterCompute.class);
  // Run internally
  Iterable<String> results = InternalVertexRunner.run(conf, graph);

  Map<Long, Double> steadyStateProbabilities =
      RandomWalkTestUtils.parseSteadyStateProbabilities(results);
  // values computed with external software
  // 0.25, 0.354872, 0.09375, 0.301377
  assertEquals(0.25, steadyStateProbabilities.get(12L), RandomWalkTestUtils.EPSILON);
  assertEquals(0.354872, steadyStateProbabilities.get(34L),
      RandomWalkTestUtils.EPSILON);
  assertEquals(0.09375, steadyStateProbabilities.get(56L), RandomWalkTestUtils.EPSILON);
  assertEquals(0.301377, steadyStateProbabilities.get(78L),
      RandomWalkTestUtils.EPSILON);
}
 

import org.apache.giraph.conf.GiraphConfiguration; //导入方法依赖的package包/类
/**
 * Run a sample BSP job locally with no vertices and make sure
 * it completes.
 *
 * @throws IOException
 * @throws ClassNotFoundException
 * @throws InterruptedException
 */
@Test
public void testEmptyVertexInputFormat()
    throws IOException, InterruptedException, ClassNotFoundException {
  GiraphConfiguration conf = new GiraphConfiguration();
  conf.setVertexClass(SimpleMsgVertex.class);
  conf.setVertexInputFormatClass(SimpleSuperstepVertexInputFormat.class);
  GiraphJob job = prepareJob(getCallingMethodName(), conf);
  job.getConfiguration().setLong(GeneratedVertexReader.READER_VERTICES, 0);
  assertTrue(job.run(true));
}
 

import org.apache.giraph.conf.GiraphConfiguration; //导入方法依赖的package包/类
@Before
public void setUp() {
  GiraphConfiguration configuration = new GiraphConfiguration();
  configuration.setVertexClass(MyVertex.class);
  conf = new ImmutableClassesGiraphConfiguration<IntWritable, IntWritable,
      NullWritable, IntWritable>(configuration);
  context = mock(Mapper.Context.class);
}
 

import org.apache.giraph.conf.GiraphConfiguration; //导入方法依赖的package包/类
@Test
public void testDifferentInputEdgesClass() throws Exception {
  String[] edges = new String[] {
      "1 2",
      "2 3",
      "2 4",
      "4 1"
  };

  GiraphConfiguration conf = new GiraphConfiguration();
  conf.setVertexClass(TestVertexCheckEdgesType.class);
  conf.setOutEdgesClass(ByteArrayEdges.class);
  conf.setInputOutEdgesClass(TestOutEdgesFilterEven.class);
  conf.setEdgeInputFormatClass(IntNullTextEdgeInputFormat.class);
  conf.setVertexOutputFormatClass(IdWithValueTextOutputFormat.class);
  Iterable<String> results = InternalVertexRunner.run(conf, null, edges);

  Map<Integer, Integer> values = parseResults(results);

  // Check that all vertices with outgoing edges in the input have been
  // created
  assertEquals(3, values.size());
  // Check the number of edges for each vertex (edges with odd target id
  // should have been removed)
  assertEquals(1, (int) values.get(1));
  assertEquals(1, (int) values.get(2));
  assertEquals(0, (int) values.get(4));
}
 

import org.apache.giraph.conf.GiraphConfiguration; //导入方法依赖的package包/类
/**
 * Set vertex edges, input format, partitioner classes and related parameters
 * based on command-line arguments.
 *
 * @param cmd           Command line arguments
 * @param configuration Giraph job configuration
 */
protected void prepareConfiguration(GiraphConfiguration configuration,
    CommandLine cmd) {
  configuration.setVertexClass(WeightedPageRankVertex.class);
  int edgesClassOption = EDGES_CLASS.getOptionIntValue(cmd, 1);
  switch (edgesClassOption) {
  case 0:
    configuration.setOutEdgesClass(LongDoubleArrayEdges.class);
    break;
  case 1:
    configuration.setOutEdgesClass(ByteArrayEdges.class);
    break;
  case 2:
    configuration.setOutEdgesClass(ByteArrayEdges.class);
    configuration.useUnsafeSerialization(true);
    break;
  case 3:
    configuration.setOutEdgesClass(ArrayListEdges.class);
    break;
  case 4:
    configuration.setOutEdgesClass(HashMapEdges.class);
    break;
  default:
    LOG.info("Unknown OutEdges class, " +
        "defaulting to LongDoubleArrayEdges");
    configuration.setOutEdgesClass(LongDoubleArrayEdges.class);
  }

  LOG.info("Using edges class " +
      GiraphConstants.VERTEX_EDGES_CLASS.get(configuration));
  if (COMBINER_TYPE.getOptionIntValue(cmd, 1) == 1) {
    configuration.setCombinerClass(DoubleSumCombiner.class);
  }

  if (EDGE_INPUT.optionTurnedOn(cmd)) {
    configuration.setEdgeInputFormatClass(PseudoRandomEdgeInputFormat.class);
  } else {
    configuration.setVertexInputFormatClass(
        PseudoRandomVertexInputFormat.class);
  }

  configuration.setLong(
      PseudoRandomInputFormatConstants.AGGREGATE_VERTICES,
      BenchmarkOption.VERTICES.getOptionLongValue(cmd));
  configuration.setLong(
      PseudoRandomInputFormatConstants.EDGES_PER_VERTEX,
      BenchmarkOption.EDGES_PER_VERTEX.getOptionLongValue(cmd));
  configuration.setFloat(
      PseudoRandomInputFormatConstants.LOCAL_EDGES_MIN_RATIO,
      BenchmarkOption.LOCAL_EDGES_MIN_RATIO.getOptionFloatValue(cmd,
          PseudoRandomInputFormatConstants.LOCAL_EDGES_MIN_RATIO_DEFAULT));

  if (OUTPUT_FORMAT.getOptionIntValue(cmd, -1) == 0) {
    LOG.info("Using vertex output format class " +
        JsonBase64VertexOutputFormat.class.getName());
    configuration.setVertexOutputFormatClass(
        JsonBase64VertexOutputFormat.class);
  }

  if (PARTITIONER.getOptionIntValue(cmd, 0) == 1) {
    configuration.setGraphPartitionerFactoryClass(
        SimpleLongRangePartitionerFactory.class);
  }

  configuration.setInt(WeightedPageRankVertex.SUPERSTEP_COUNT,
      BenchmarkOption.SUPERSTEPS.getOptionIntValue(cmd));
}