Java GiraphConfiguration.setOutEdgesClass方法代码示例

import org.apache.giraph.conf.GiraphConfiguration; //导入方法依赖的package包/类
@Test
public void testEdgesOnly() throws Exception {
  String[] edges = new String[] {
      "1 2",
      "2 3",
      "2 4",
      "4 1"
  };

  GiraphConfiguration conf = new GiraphConfiguration();
  conf.setVertexClass(TestVertexWithNumEdges.class);
  conf.setOutEdgesClass(ByteArrayEdges.class);
  conf.setEdgeInputFormatClass(IntNullTextEdgeInputFormat.class);
  conf.setVertexOutputFormatClass(IdWithValueTextOutputFormat.class);
  Iterable<String> results = InternalVertexRunner.run(conf, null, edges);

  Map<Integer, Integer> values = parseResults(results);

  // Check that all vertices with outgoing edges have been created
  assertEquals(3, values.size());
  // Check the number of edges for each vertex
  assertEquals(1, (int) values.get(1));
  assertEquals(2, (int) values.get(2));
  assertEquals(1, (int) values.get(4));
}
 

import org.apache.giraph.conf.GiraphConfiguration; //导入方法依赖的package包/类
@Override
protected void prepareConfiguration(GiraphConfiguration conf,
    CommandLine cmd) {
  conf.setVertexClass(PageRankVertex.class);
  conf.setOutEdgesClass(IntNullArrayEdges.class);
  conf.setCombinerClass(FloatSumCombiner.class);
  conf.setVertexInputFormatClass(
      PseudoRandomIntNullVertexInputFormat.class);

  conf.setInt(PseudoRandomInputFormatConstants.AGGREGATE_VERTICES,
      BenchmarkOption.VERTICES.getOptionIntValue(cmd));
  conf.setInt(PseudoRandomInputFormatConstants.EDGES_PER_VERTEX,
      BenchmarkOption.EDGES_PER_VERTEX.getOptionIntValue(cmd));
  conf.setInt(PageRankVertex.SUPERSTEP_COUNT,
      BenchmarkOption.SUPERSTEPS.getOptionIntValue(cmd));
  conf.setFloat(PseudoRandomInputFormatConstants.LOCAL_EDGES_MIN_RATIO,
      BenchmarkOption.LOCAL_EDGES_MIN_RATIO.getOptionFloatValue(cmd,
          PseudoRandomInputFormatConstants.LOCAL_EDGES_MIN_RATIO_DEFAULT));
}
 

import org.apache.giraph.conf.GiraphConfiguration; //导入方法依赖的package包/类
private void runJob(String tableName, GiraphConfiguration conf) throws Exception {
  String[] edges = new String[] {
      "1 2",
      "2 3",
      "2 4",
      "4 1"
  };

  GiraphHiveConstants.HIVE_VERTEX_OUTPUT_TABLE.set(conf, tableName);
  GiraphHiveConstants.VERTEX_TO_HIVE_CLASS.set(conf, HiveOutputIntIntVertex.class);

  conf.setComputationClass(ComputationCountEdges.class);
  conf.setOutEdgesClass(ByteArrayEdges.class);
  conf.setEdgeInputFormatClass(IntNullTextEdgeInputFormat.class);
  conf.setVertexOutputFormatClass(HiveVertexOutputFormat.class);
  InternalVertexRunner.run(conf, null, edges);

  Helpers.commitJob(conf);
}
 

import org.apache.giraph.conf.GiraphConfiguration; //导入方法依赖的package包/类
@Test
public void testCheckFailsJob() throws Exception {
  String tableName = "test1";
  hiveServer.createTable("CREATE TABLE " + tableName +
     " (i1 INT, i2 BIGINT) ");

  GiraphConfiguration conf = new GiraphConfiguration();
  String[] edges = new String[] {
      "1 2",
      "2 3",
      "2 4",
      "4 1"
  };

  GiraphHiveConstants.HIVE_VERTEX_OUTPUT_TABLE.set(conf, tableName);
  GiraphHiveConstants.VERTEX_TO_HIVE_CLASS.set(conf, HiveOutputIntIntVertex.class);

  conf.setComputationClass(ComputationCountEdges.class);
  conf.setOutEdgesClass(ByteArrayEdges.class);
  conf.setEdgeInputFormatClass(IntNullTextEdgeInputFormat.class);
  conf.setVertexOutputFormatClass(HiveVertexOutputFormat.class);
  try {
    Iterable<String> result = InternalVertexRunner.run(conf, null, edges);
    assertNull(result);
  } catch (IllegalArgumentException e) { }
}
 

import org.apache.giraph.conf.GiraphConfiguration; //导入方法依赖的package包/类
@Override
protected void prepareConfiguration(GiraphConfiguration conf,
    CommandLine cmd) {
  conf.setComputationClass(ShortestPathsComputation.class);
  if (EDGES_CLASS.getOptionIntValue(cmd, 1) == 1) {
    conf.setOutEdgesClass(ArrayListEdges.class);
  } else {
    conf.setOutEdgesClass(HashMapEdges.class);
  }
  LOG.info("Using class " + GiraphConstants.COMPUTATION_CLASS.get(conf));
  conf.setVertexInputFormatClass(PseudoRandomVertexInputFormat.class);
  if (!NO_COMBINER.optionTurnedOn(cmd)) {
    conf.setCombinerClass(MinimumDoubleCombiner.class);
  }
  conf.setLong(PseudoRandomInputFormatConstants.AGGREGATE_VERTICES,
      BenchmarkOption.VERTICES.getOptionLongValue(cmd));
  conf.setLong(PseudoRandomInputFormatConstants.EDGES_PER_VERTEX,
      BenchmarkOption.EDGES_PER_VERTEX.getOptionLongValue(cmd));
}
 

import org.apache.giraph.conf.GiraphConfiguration; //导入方法依赖的package包/类
@Test
public void testGetsCalled() throws Exception {
  assertEquals(0, Obs.postApp);

  String[] graph = new String[] { "1", "2", "3" };

  String klasses[] = new String[] {
      Obs.class.getName(),
      Obs.class.getName()
  };

  GiraphConfiguration conf = new GiraphConfiguration();
  conf.set(GiraphConstants.MASTER_OBSERVER_CLASSES.getKey(),
      arrayToString(klasses));
  conf.setComputationClass(SimpleComputation.class);
  conf.setOutEdgesClass(ByteArrayEdges.class);
  conf.setVertexInputFormatClass(InputFormat.class);
  InternalVertexRunner.run(conf, graph);

  assertEquals(2, Obs.preApp);
  // 3 supersteps + 1 input superstep * 2 observers = 8 callbacks
  assertEquals(8, Obs.preSuperstep);
  assertEquals(8, Obs.postSuperstep);
  assertEquals(2, Obs.postApp);
}
 

import org.apache.giraph.conf.GiraphConfiguration; //导入方法依赖的package包/类
private Vertex<LongWritable, FloatWritable, DoubleWritable, LongWritable>
instantiateVertex(Class<? extends OutEdges> edgesClass) {
  GiraphConfiguration giraphConfiguration = new GiraphConfiguration();
  giraphConfiguration.setVertexClass(TestVertex.class);
  giraphConfiguration.setOutEdgesClass(edgesClass);
  ImmutableClassesGiraphConfiguration immutableClassesGiraphConfiguration =
      new ImmutableClassesGiraphConfiguration(giraphConfiguration);
  return immutableClassesGiraphConfiguration.createVertex();
}
 

import org.apache.giraph.conf.GiraphConfiguration; //导入方法依赖的package包/类
/**
 * A local integration test on toy data
 */
@Test
public void testToyData() throws Exception {

  // a small four vertex graph
  String[] graph = new String[] {
      "[1,0,[[2,1],[3,3]]]",
      "[2,0,[[3,1],[4,10]]]",
      "[3,0,[[4,2]]]",
      "[4,0,[]]"
  };

  GiraphConfiguration conf = new GiraphConfiguration();
  // start from vertex 1
  SOURCE_ID.set(conf, 1);
  conf.setComputationClass(SimpleShortestPathsComputation.class);
  conf.setOutEdgesClass(ByteArrayEdges.class);
  conf.setVertexInputFormatClass(
      JsonLongDoubleFloatDoubleVertexInputFormat.class);
  conf.setVertexOutputFormatClass(
      JsonLongDoubleFloatDoubleVertexOutputFormat.class);

  // run internally
  Iterable<String> results = InternalVertexRunner.run(conf, graph);

  Map<Long, Double> distances = parseDistances(results);

  // verify results
  assertNotNull(distances);
  assertEquals(4, (int) distances.size());
  assertEquals(0.0, (double) distances.get(1L), 0d);
  assertEquals(1.0, (double) distances.get(2L), 0d);
  assertEquals(2.0, (double) distances.get(3L), 0d);
  assertEquals(4.0, (double) distances.get(4L), 0d);
}
 

import org.apache.giraph.conf.GiraphConfiguration; //导入方法依赖的package包/类
/**
 * A local integration test on toy data
 */
@Test
public void testToyData() throws Exception {
  // A small graph
  String[] graph = new String[] { "12 34 56", "34 78", "56 34 78", "78 34" };

  GiraphConfiguration conf = new GiraphConfiguration();
  conf.setInt(RandomWalkWithRestartComputation.SOURCE_VERTEX, 12);
  conf.setInt(RandomWalkWithRestartComputation.MAX_SUPERSTEPS, 30);
  conf.setFloat(
      RandomWalkWithRestartComputation.TELEPORTATION_PROBABILITY, 0.25f);
  conf.setComputationClass(RandomWalkWithRestartComputation.class);
  conf.setOutEdgesClass(ByteArrayEdges.class);
  conf.setVertexInputFormatClass(LongDoubleDoubleTextInputFormat.class);
  conf.setVertexOutputFormatClass(
      VertexWithDoubleValueDoubleEdgeTextOutputFormat.class);
  conf.setWorkerContextClass(RandomWalkWorkerContext.class);
  conf.setMasterComputeClass(RandomWalkVertexMasterCompute.class);
  // Run internally
  Iterable<String> results = InternalVertexRunner.run(conf, graph);

  Map<Long, Double> steadyStateProbabilities =
      RandomWalkTestUtils.parseSteadyStateProbabilities(results);
  // values computed with external software
  // 0.25, 0.354872, 0.09375, 0.301377
  assertEquals(0.25, steadyStateProbabilities.get(12L), RandomWalkTestUtils.EPSILON);
  assertEquals(0.354872, steadyStateProbabilities.get(34L),
      RandomWalkTestUtils.EPSILON);
  assertEquals(0.09375, steadyStateProbabilities.get(56L), RandomWalkTestUtils.EPSILON);
  assertEquals(0.301377, steadyStateProbabilities.get(78L),
      RandomWalkTestUtils.EPSILON);
}
 

import org.apache.giraph.conf.GiraphConfiguration; //导入方法依赖的package包/类
/**
 * A local integration test on toy data
 */
@Test
public void testWeightedGraph() throws Exception {
  // A small graph
  String[] graph =
      new String[] { "12 34:0.1 56:0.9", "34 78:0.9 56:0.1",
        "56 12:0.1 34:0.8 78:0.1", "78 34:1.0" };

  GiraphConfiguration conf = new GiraphConfiguration();
  conf.setInt(RandomWalkWithRestartComputation.SOURCE_VERTEX, 12);
  conf.setInt(RandomWalkWithRestartComputation.MAX_SUPERSTEPS, 30);
  conf.setFloat(
      RandomWalkWithRestartComputation.TELEPORTATION_PROBABILITY, 0.15f);
  conf.setComputationClass(RandomWalkWithRestartComputation.class);
  conf.setOutEdgesClass(ByteArrayEdges.class);
  conf.setVertexInputFormatClass(
      NormalizingLongDoubleDoubleTextInputFormat.class);
  conf.setVertexOutputFormatClass(
      VertexWithDoubleValueDoubleEdgeTextOutputFormat.class);
  conf.setWorkerContextClass(RandomWalkWorkerContext.class);
  conf.setMasterComputeClass(RandomWalkVertexMasterCompute.class);
  // Run internally
  Iterable<String> results = InternalVertexRunner.run(conf, graph);

  Map<Long, Double> steadyStateProbabilities =
      RandomWalkTestUtils.parseSteadyStateProbabilities(results);
  // values computed with external software
  // 0.163365, 0.378932, 0.156886, 0.300816
  assertEquals(0.163365, steadyStateProbabilities.get(12L),
      RandomWalkTestUtils.EPSILON);
  assertEquals(0.378932, steadyStateProbabilities.get(34L),
      RandomWalkTestUtils.EPSILON);
  assertEquals(0.156886, steadyStateProbabilities.get(56L),
      RandomWalkTestUtils.EPSILON);
  assertEquals(0.300816, steadyStateProbabilities.get(78L),
      RandomWalkTestUtils.EPSILON);
}
 

import org.apache.giraph.conf.GiraphConfiguration; //导入方法依赖的package包/类
public static OutEdges
instantiateOutEdges(Class<? extends OutEdges> edgesClass) {
  GiraphConfiguration giraphConfiguration = new GiraphConfiguration();
  // Needed to extract type arguments in ReflectionUtils.
  giraphConfiguration.setVertexClass(TestVertex.class);
  giraphConfiguration.setOutEdgesClass(edgesClass);
  ImmutableClassesGiraphConfiguration immutableClassesGiraphConfiguration =
      new ImmutableClassesGiraphConfiguration(giraphConfiguration);
  return immutableClassesGiraphConfiguration.createOutEdges();
}
 

import org.apache.giraph.conf.GiraphConfiguration; //导入方法依赖的package包/类
/**
 * A local integration test on toy data
 */
@Test
public void testWeightedGraph() throws Exception {
  // A small graph
  String[] graph =
      new String[] { "12 34:0.1 56:0.9", "34 78:0.9 56:0.1",
        "56 12:0.1 34:0.8 78:0.1", "78 34:1.0" };

  GiraphConfiguration conf = new GiraphConfiguration();
  conf.setInt(RandomWalkWithRestartVertex.SOURCE_VERTEX, 12);
  conf.setInt(RandomWalkWithRestartVertex.MAX_SUPERSTEPS, 30);
  conf.setFloat(RandomWalkWithRestartVertex.TELEPORTATION_PROBABILITY, 0.15f);
  conf.setVertexClass(RandomWalkWithRestartVertex.class);
  conf.setOutEdgesClass(ByteArrayEdges.class);
  conf.setVertexInputFormatClass(
      NormalizingLongDoubleDoubleTextInputFormat.class);
  conf.setVertexOutputFormatClass(
      VertexWithDoubleValueDoubleEdgeTextOutputFormat.class);
  conf.setWorkerContextClass(RandomWalkWorkerContext.class);
  conf.setMasterComputeClass(RandomWalkVertexMasterCompute.class);
  // Run internally
  Iterable<String> results = InternalVertexRunner.run(conf, graph);

  Map<Long, Double> steadyStateProbabilities =
      RandomWalkTestUtils.parseSteadyStateProbabilities(results);
  // values computed with external software
  // 0.163365, 0.378932, 0.156886, 0.300816
  assertEquals(0.163365, steadyStateProbabilities.get(12L),
      RandomWalkTestUtils.EPSILON);
  assertEquals(0.378932, steadyStateProbabilities.get(34L),
      RandomWalkTestUtils.EPSILON);
  assertEquals(0.156886, steadyStateProbabilities.get(56L),
      RandomWalkTestUtils.EPSILON);
  assertEquals(0.300816, steadyStateProbabilities.get(78L),
      RandomWalkTestUtils.EPSILON);
}
 

import org.apache.giraph.conf.GiraphConfiguration; //导入方法依赖的package包/类
@Override
protected void prepareConfiguration(GiraphConfiguration conf,
    CommandLine cmd) {
  if (BenchmarkOption.JYTHON.optionTurnedOn(cmd)) {
    GiraphTypes types = new GiraphTypes();
    types.inferFrom(PageRankComputation.class);

    String script;
    DeployType deployType;
    if (BenchmarkOption.SCRIPT_PATH.optionTurnedOn(cmd)) {
      deployType = DeployType.DISTRIBUTED_CACHE;
      String path = BenchmarkOption.SCRIPT_PATH.getOptionValue(cmd);
      Path hadoopPath = new Path(path);
      Path remotePath = DistributedCacheUtils.copyAndAdd(hadoopPath, conf);
      script = remotePath.toString();
    } else {
      deployType = DeployType.RESOURCE;
      script = ReflectionUtils.getPackagePath(this) + "/page-rank.py";
    }
    ScriptLoader.setScriptsToLoad(conf, script, deployType, Language.JYTHON);
    types.writeIfUnset(conf);
    JythonUtils.init(conf, "PageRank");
  } else {
    conf.setComputationClass(PageRankComputation.class);
  }
  conf.setOutEdgesClass(IntNullArrayEdges.class);
  conf.setCombinerClass(FloatSumCombiner.class);
  conf.setVertexInputFormatClass(
      PseudoRandomIntNullVertexInputFormat.class);

  conf.setInt(PseudoRandomInputFormatConstants.AGGREGATE_VERTICES,
      BenchmarkOption.VERTICES.getOptionIntValue(cmd));
  conf.setInt(PseudoRandomInputFormatConstants.EDGES_PER_VERTEX,
      BenchmarkOption.EDGES_PER_VERTEX.getOptionIntValue(cmd));
  conf.setInt(PageRankComputation.SUPERSTEP_COUNT,
      BenchmarkOption.SUPERSTEPS.getOptionIntValue(cmd));
  conf.setFloat(PseudoRandomInputFormatConstants.LOCAL_EDGES_MIN_RATIO,
      BenchmarkOption.LOCAL_EDGES_MIN_RATIO.getOptionFloatValue(cmd,
          PseudoRandomInputFormatConstants.LOCAL_EDGES_MIN_RATIO_DEFAULT));
}
 

import org.apache.giraph.conf.GiraphConfiguration; //导入方法依赖的package包/类
@Test
public void testDifferentInputEdgesClass() throws Exception {
  String[] edges = new String[] {
      "1 2",
      "2 3",
      "2 4",
      "4 1"
  };

  GiraphConfiguration conf = new GiraphConfiguration();
  conf.setComputationClass(TestComputationCheckEdgesType.class);
  conf.setOutEdgesClass(ByteArrayEdges.class);
  conf.setInputOutEdgesClass(TestOutEdgesFilterEven.class);
  conf.setEdgeInputFormatClass(IntNullTextEdgeInputFormat.class);
  conf.setVertexOutputFormatClass(IdWithValueTextOutputFormat.class);
  Iterable<String> results = InternalVertexRunner.run(conf, null, edges);

  Map<Integer, Integer> values = parseResults(results);

  // Check that all vertices with outgoing edges in the input have been
  // created
  assertEquals(3, values.size());
  // Check the number of edges for each vertex (edges with odd target id
  // should have been removed)
  assertEquals(1, (int) values.get(1));
  assertEquals(1, (int) values.get(2));
  assertEquals(0, (int) values.get(4));
}
 

import org.apache.giraph.conf.GiraphConfiguration; //导入方法依赖的package包/类
/**
   * prepare a vertex and computation for use in a unit test by setting its
   * internal state and injecting mocked dependencies,
   *
   * @param vertex Vertex
   * @param vertexId initial vertex id
   * @param vertexValue initial vertex value
   * @param isHalted initial halted state of the vertex
   * @param computation Computation
   * @param superstep Superstep
   * @param <I> vertex id
   * @param <V> vertex data
   * @param <E> edge data
   * @param <M> message data
   * @return
   * @throws Exception
   */
public static <I extends WritableComparable, V extends Writable,
    E extends Writable, M extends Writable>
MockedEnvironment<I, V, E, M> prepareVertexAndComputation(
    Vertex<I, V, E> vertex, I vertexId, V vertexValue, boolean isHalted,
    Computation<I, V, E, M, M> computation, long superstep) throws
    Exception {
  MockedEnvironment<I, V, E, M> env = new MockedEnvironment<I, V, E, M>();
  Mockito.when(env.getGraphState().getSuperstep()).thenReturn(superstep);
  Mockito.when(env.getGraphState().getContext())
      .thenReturn(env.getContext());
  Mockito.when(env.getContext().getConfiguration())
      .thenReturn(env.getConfiguration());
  computation.initialize(env.getGraphState(),
      env.getWorkerClientRequestProcessor(), null, null, null);

  GiraphConfiguration giraphConf = new GiraphConfiguration();
  giraphConf.setComputationClass(computation.getClass());
  giraphConf.setOutEdgesClass(ArrayListEdges.class);
  ImmutableClassesGiraphConfiguration<I, V, E> conf =
      new ImmutableClassesGiraphConfiguration<I, V, E>(giraphConf);
  computation.setConf(conf);

  vertex.setConf(conf);
  vertex.initialize(vertexId, vertexValue);
  if (isHalted) {
    vertex.voteToHalt();
  }

  return env;
}