Java GiraphConfiguration.setEdgeInputFormatClass方法代码示例

import org.apache.giraph.conf.GiraphConfiguration; //导入方法依赖的package包/类
@Test
public void testMax() throws Exception {

    Vertex v1 = graph.addVertex(T.id, 1, T.label, "hi");
    Vertex v2 = graph.addVertex(T.id, 2, T.label, "world");
    Vertex v5 = graph.addVertex(T.id, 5, T.label, "bye");
    v5.addEdge("e", v1);
    v1.addEdge("e", v5);
    v1.addEdge("e", v2);
    v2.addEdge("e", v5);

    HBaseGraphConfiguration hconf = graph.configuration();
    GiraphConfiguration conf = new GiraphConfiguration(hconf.toHBaseConfiguration());
    conf.setComputationClass(MaxComputation.class);
    conf.setEdgeInputFormatClass(HBaseEdgeInputFormat.class);
    conf.setVertexInputFormatClass(HBaseVertexInputFormat.class);
    conf.setVertexOutputFormatClass(IdWithValueTextOutputFormat.class);

    Iterable<String> results = InternalHBaseVertexRunner.run(conf);

    Map<Integer, Integer> values = parseResults(results);
    assertEquals(3, values.size());
    assertEquals(5, (int) values.get(1));
    assertEquals(5, (int) values.get(2));
    assertEquals(5, (int) values.get(5));
}
 

import org.apache.giraph.conf.GiraphConfiguration; //导入方法依赖的package包/类
@Test
public void testMax() throws Exception {
    HBaseGraphConfiguration hconf = graph.configuration();

    GiraphConfiguration conf = new GiraphConfiguration(hconf.toHBaseConfiguration());
    conf.setComputationClass(MaxComputation.class);
    conf.setEdgeInputFormatClass(HBaseEdgeInputFormat.class);
    conf.setVertexInputFormatClass(HBaseVertexInputFormat.class);
    conf.setEdgeOutputFormatClass(CreateEdgeOutputFormat.class);
    conf.setVertexOutputFormatClass(MaxPropertyVertexOutputFormat.class);

    Vertex v1 = graph.addVertex(T.id, 1, T.label, "hi");
    Vertex v2 = graph.addVertex(T.id, 2, T.label, "world");
    Vertex v5 = graph.addVertex(T.id, 5, T.label, "bye");
    v5.addEdge("e", v1);
    v1.addEdge("e", v5);
    v1.addEdge("e", v2);
    v2.addEdge("e", v5);

    InternalHBaseVertexRunner.run(conf);

    graph.vertices().forEachRemaining(v -> assertEquals(5, v.property("max").value()));
    assertEquals(4, IteratorUtils.count(IteratorUtils.filter(graph.edges(), e -> e.label().equals("e2"))));
}
 

import org.apache.giraph.conf.GiraphConfiguration; //导入方法依赖的package包/类
@Test
public void testEdgesOnly() throws Exception {
  String[] edges = new String[] {
      "1 2",
      "2 3",
      "2 4",
      "4 1"
  };

  GiraphConfiguration conf = new GiraphConfiguration();
  conf.setVertexClass(TestVertexWithNumEdges.class);
  conf.setOutEdgesClass(ByteArrayEdges.class);
  conf.setEdgeInputFormatClass(IntNullTextEdgeInputFormat.class);
  conf.setVertexOutputFormatClass(IdWithValueTextOutputFormat.class);
  Iterable<String> results = InternalVertexRunner.run(conf, null, edges);

  Map<Integer, Integer> values = parseResults(results);

  // Check that all vertices with outgoing edges have been created
  assertEquals(3, values.size());
  // Check the number of edges for each vertex
  assertEquals(1, (int) values.get(1));
  assertEquals(2, (int) values.get(2));
  assertEquals(1, (int) values.get(4));
}
 

import org.apache.giraph.conf.GiraphConfiguration; //导入方法依赖的package包/类
@Test
public void testEdgesOnlyWithReverse() throws Exception {
  String[] edges = new String[] {
      "1 2",
      "2 3",
      "2 4",
      "4 1"
  };

  GiraphConfiguration conf = new GiraphConfiguration();
  conf.setVertexClass(TestVertexWithNumEdges.class);
  conf.setOutEdgesClass(ByteArrayEdges.class);
  conf.setEdgeInputFormatClass(IntNullReverseTextEdgeInputFormat.class);
  conf.setVertexOutputFormatClass(IdWithValueTextOutputFormat.class);
  Iterable<String> results = InternalVertexRunner.run(conf, null, edges);

  Map<Integer, Integer> values = parseResults(results);

  // Check that all vertices with outgoing edges have been created
  assertEquals(4, values.size());
  // Check the number of edges for each vertex
  assertEquals(2, (int) values.get(1));
  assertEquals(3, (int) values.get(2));
  assertEquals(1, (int) values.get(3));
  assertEquals(2, (int) values.get(4));
}
 

import org.apache.giraph.conf.GiraphConfiguration; //导入方法依赖的package包/类
private void runJob(String tableName, GiraphConfiguration conf) throws Exception {
  String[] edges = new String[] {
      "1 2",
      "2 3",
      "2 4",
      "4 1"
  };

  GiraphHiveConstants.HIVE_VERTEX_OUTPUT_TABLE.set(conf, tableName);
  GiraphHiveConstants.VERTEX_TO_HIVE_CLASS.set(conf, HiveOutputIntIntVertex.class);

  conf.setComputationClass(ComputationCountEdges.class);
  conf.setOutEdgesClass(ByteArrayEdges.class);
  conf.setEdgeInputFormatClass(IntNullTextEdgeInputFormat.class);
  conf.setVertexOutputFormatClass(HiveVertexOutputFormat.class);
  InternalVertexRunner.run(conf, null, edges);

  Helpers.commitJob(conf);
}
 

import org.apache.giraph.conf.GiraphConfiguration; //导入方法依赖的package包/类
@Test
public void testCheckJobThrows() throws Exception {
  String tableName = "test1";
  hiveServer.createTable("CREATE TABLE " + tableName +
      " (i1 BIGINT, i2 INT) " +
      " ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t'");
  String[] rows = {
      "1\t2",
      "2\t3",
      "2\t4",
      "4\t1",
  };
  hiveServer.loadData(tableName, rows);

  GiraphConfiguration conf = new GiraphConfiguration();
  HIVE_EDGE_INPUT.setTable(conf, tableName);
  HIVE_EDGE_INPUT.setClass(conf, HiveIntNullEdge.class);
  conf.setComputationClass(ComputationCountEdges.class);
  conf.setEdgeInputFormatClass(HiveEdgeInputFormat.class);
  conf.setVertexOutputFormatClass(IdWithValueTextOutputFormat.class);
  assertNull(InternalVertexRunner.run(conf, new String[0], new String[0]));
}
 

import org.apache.giraph.conf.GiraphConfiguration; //导入方法依赖的package包/类
@Test
public void testEdgeFilter() throws Exception {
  String[] edges = new String[] {
      "1 2",
      "2 3",
      "2 4",
      "4 1"
  };

  GiraphConfiguration conf = new GiraphConfiguration();
  conf.setComputationClass(ComputationCountEdges.class);
  conf.setEdgeInputFormatClass(IntNullTextEdgeInputFormat.class);
  conf.setEdgeInputFilterClass(EdgeFilter.class);
  conf.setVertexOutputFormatClass(IdWithValueTextOutputFormat.class);
  Iterable<String> results = InternalVertexRunner.run(conf, null, edges);

  Map<Integer, Integer> values = parseResults(results);

  assertEquals(2, values.size());
  assertEquals(1, (int) values.get(1));
  assertEquals(1, (int) values.get(4));
}
 

import org.apache.giraph.conf.GiraphConfiguration; //导入方法依赖的package包/类
@Test
public void testEdgesOnly() throws Exception {
  String[] edges = new String[] {
      "1 2",
      "2 3",
      "2 4",
      "4 1"
  };

  GiraphConfiguration conf = new GiraphConfiguration();
  conf.setComputationClass(ComputationCountEdges.class);
  conf.setOutEdgesClass(ByteArrayEdges.class);
  conf.setEdgeInputFormatClass(IntNullTextEdgeInputFormat.class);
  conf.setVertexOutputFormatClass(IdWithValueTextOutputFormat.class);
  Iterable<String> results = InternalVertexRunner.run(conf, null, edges);

  Map<Integer, Integer> values = parseResults(results);

  // Check that all vertices with outgoing edges have been created
  assertEquals(3, values.size());
  // Check the number of edges for each vertex
  assertEquals(1, (int) values.get(1));
  assertEquals(2, (int) values.get(2));
  assertEquals(1, (int) values.get(4));
}
 

import org.apache.giraph.conf.GiraphConfiguration; //导入方法依赖的package包/类
@Test
public void getEmptyDb() throws Exception {
  Iterable<String>    results;
  Iterator<String>    result;
  GiraphConfiguration conf    = new GiraphConfiguration();
  GIRAPH_GORA_DATASTORE_CLASS.
  set(conf, "org.apache.gora.memory.store.MemStore");
  GIRAPH_GORA_KEYS_FACTORY_CLASS.
  set(conf,"org.apache.giraph.io.gora.utils.DefaultKeyFactory");
  GIRAPH_GORA_KEY_CLASS.set(conf,"java.lang.String");
  GIRAPH_GORA_PERSISTENT_CLASS.
  set(conf,"org.apache.giraph.io.gora.generated.GEdge");
  GIRAPH_GORA_START_KEY.set(conf,"1");
  GIRAPH_GORA_END_KEY.set(conf,"3");
  conf.set("io.serializations",
      "org.apache.hadoop.io.serializer.WritableSerialization," +
      "org.apache.hadoop.io.serializer.JavaSerialization");
  conf.setComputationClass(EmptyComputation.class);
  conf.setEdgeInputFormatClass(GoraGEdgeEdgeInputFormat.class);
  results = InternalVertexRunner.run(conf, new String[0], new String[0]);
  Assert.assertNotNull(results);
  result = results.iterator();
  Assert.assertFalse(result.hasNext());
}
 

import org.apache.giraph.conf.GiraphConfiguration; //导入方法依赖的package包/类
/**
 * Initialize hive input settings
 *
 * @param conf Configuration to write to
 * @throws TException thrift problem
 */
private void setupHiveInputs(GiraphConfiguration conf) throws TException {
  if (hiveToVertexClass != null) {
    hiveVertexInputDescription.setNumSplits(HIVE_VERTEX_SPLITS.get(conf));
    HiveApiInputFormat.setProfileInputDesc(conf, hiveVertexInputDescription,
        VERTEX_INPUT_PROFILE_ID);
    conf.setVertexInputFormatClass(HiveVertexInputFormat.class);
    HiveTableSchemas.put(conf, VERTEX_INPUT_PROFILE_ID,
        hiveVertexInputDescription.hiveTableName());
  }

  if (hiveToEdgeClass != null) {
    hiveEdgeInputDescription.setNumSplits(HIVE_EDGE_SPLITS.get(conf));
    HiveApiInputFormat.setProfileInputDesc(conf, hiveEdgeInputDescription,
        EDGE_INPUT_PROFILE_ID);
    conf.setEdgeInputFormatClass(HiveEdgeInputFormat.class);
    HiveTableSchemas.put(conf, EDGE_INPUT_PROFILE_ID,
        hiveEdgeInputDescription.hiveTableName());
  }
}
 

import org.apache.giraph.conf.GiraphConfiguration; //导入方法依赖的package包/类
/**
 * A local integration test on toy data
 */
@Test
public void testToyData() throws Exception {

    Vertex v1 = graph.addVertex(T.id, 1);
    Vertex v2 = graph.addVertex(T.id, 2);
    Vertex v3 = graph.addVertex(T.id, 3);
    Vertex v4 = graph.addVertex(T.id, 4);
    v1.addEdge("e", v2, "weight", 1.0);
    v1.addEdge("e", v3, "weight", 3.0);
    v2.addEdge("e", v3, "weight", 1.0);
    v2.addEdge("e", v4, "weight", 10.0);
    v3.addEdge("e", v4, "weight", 2.0);

    HBaseGraphConfiguration hconf = graph.configuration();
    GiraphConfiguration conf = new GiraphConfiguration(hconf.toHBaseConfiguration());
    // start from vertex 1
    SOURCE_ID.set(conf, 1);
    conf.setComputationClass(SimpleShortestPathsComputation.class);
    conf.setEdgeInputFormatClass(HBaseEdgeInputFormat.class);
    conf.setVertexInputFormatClass(HBaseVertexInputFormat.class);
    conf.setVertexOutputFormatClass(VertexWithDoubleValueNullEdgeTextOutputFormat.class);

    // run internally
    Iterable<String> results = InternalHBaseVertexRunner.run(conf);

    Map<Long, Double> distances = parseDistances(results);

    // verify results
    assertNotNull(distances);
    assertEquals(4, distances.size());
    assertEquals(0.0, distances.get(1L), 0d);
    assertEquals(1.0, distances.get(2L), 0d);
    assertEquals(2.0, distances.get(3L), 0d);
    assertEquals(4.0, distances.get(4L), 0d);
}
 

import org.apache.giraph.conf.GiraphConfiguration; //导入方法依赖的package包/类
@Test
public void testToyData2() throws Exception {

  // A small graph
  String[] vertices = new String[]{
          "a 1",
          "b 1",
  };

  String[] edges = new String[]{
          "a b",
          "b a",
  };

  GiraphConfiguration conf = new GiraphConfiguration();
  conf.setComputationClass(LinkRankComputation.class);
  conf.setOutEdgesClass(ByteArrayEdges.class);

  conf.setVertexInputFormatClass(LinkRankVertexInputFormat.class);
  conf.setVertexOutputFormatClass(
          LinkRankVertexOutputFormat.class);
  conf.setEdgeInputFormatClass(LinkRankEdgeInputFormat.class);
  conf.setInt("giraph.linkRank.superstepCount", 10);
  conf.setInt("giraph.linkRank.scale", 10);
  conf.setMasterComputeClass(LinkRankVertexMasterCompute.class);
  // Run internally
  Iterable<String> results = InternalVertexRunner.run(conf, vertices, edges);

  HashMap<String, Double> hm = new HashMap();
  for (String result : results) {
    String[] tokens = result.split("\t");
    hm.put(tokens[0], Double.parseDouble(tokens[1]));
    LOG.info(result);
  }

  assertEquals("a scores are not the same", hm.get("a"), 5.0d, DELTA);
  assertEquals("b scores are not the same", hm.get("b"), 5.0d, DELTA);
}
 

import org.apache.giraph.conf.GiraphConfiguration; //导入方法依赖的package包/类
@Test
public void testEdgeInput() throws Exception {
  String tableName = "test1";
  hiveServer.createTable("CREATE TABLE " + tableName +
      " (i1 INT, i2 INT) " +
      " ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t'");
  String[] rows = {
      "1\t2",
      "2\t3",
      "2\t4",
      "4\t1",
  };
  hiveServer.loadData(tableName, rows);

  GiraphConfiguration conf = new GiraphConfiguration();
  HIVE_EDGE_INPUT.setTable(conf, tableName);
  HIVE_EDGE_INPUT.setClass(conf, HiveIntNullEdge.class);
  conf.setComputationClass(ComputationCountEdges.class);
  conf.setEdgeInputFormatClass(HiveEdgeInputFormat.class);
  conf.setVertexOutputFormatClass(IdWithValueTextOutputFormat.class);
  Iterable<String> output = InternalVertexRunner.run(conf, new String[0], new String[0]);

  Map<Integer, Integer> data = Helpers.parseIntIntResults(output);
  assertEquals(3, data.size());
  assertEquals(1, (int) data.get(1));
  assertEquals(2, (int) data.get(2));
  assertEquals(1, (int) data.get(4));
}
 

import org.apache.giraph.conf.GiraphConfiguration; //导入方法依赖的package包/类
@Test
public void testEdgeInputWithPartitions() throws Exception {
  String tableName = "test1";
  String partition = "ds='foobar'";
  hiveServer.createTable("CREATE TABLE " + tableName +
      " (i1 INT, i2 INT) " +
      " PARTITIONED BY (ds STRING) " +
      " ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t' ");
  String[] rows = {
      "1\t2",
      "2\t3",
      "2\t4",
      "4\t1",
  };
  hiveServer.loadData(tableName, partition, rows);

  GiraphConfiguration conf = new GiraphConfiguration();
  HIVE_EDGE_INPUT.setTable(conf, tableName);
  HIVE_EDGE_INPUT.setPartition(conf, partition);
  HIVE_EDGE_INPUT.setClass(conf, HiveIntNullEdge.class);
  conf.setComputationClass(ComputationCountEdges.class);
  conf.setEdgeInputFormatClass(HiveEdgeInputFormat.class);
  conf.setVertexOutputFormatClass(IdWithValueTextOutputFormat.class);
  Iterable<String> output = InternalVertexRunner.run(conf, new String[0], new String[0]);

  Map<Integer, Integer> data = Helpers.parseIntIntResults(output);
  assertEquals(3, data.size());
  assertEquals(1, (int) data.get(1));
  assertEquals(2, (int) data.get(2));
  assertEquals(1, (int) data.get(4));
}
 

import org.apache.giraph.conf.GiraphConfiguration; //导入方法依赖的package包/类
@Test
public void testEdgeInputWithValues() throws Exception {
  String tableName = "test1";
  hiveServer.createTable("CREATE TABLE " + tableName +
      " (i1 INT, i2 INT, d3 DOUBLE) " +
      " ROW FORMAT DELIMITED " +
      " FIELDS TERMINATED BY '\t' " +
      " COLLECTION ITEMS TERMINATED BY ',' ");
  String[] rows = {
      "1\t2\t0.22",
      "2\t3\t0.33",
      "2\t4\t0.44",
      "4\t1\t0.11",
  };
  hiveServer.loadData(tableName, rows);

  GiraphConfiguration conf = new GiraphConfiguration();
  HIVE_EDGE_INPUT.setTable(conf, tableName);
  HIVE_EDGE_INPUT.setClass(conf, HiveIntDoubleEdge.class);
  conf.setComputationClass(ComputationSumEdges.class);
  conf.setEdgeInputFormatClass(HiveEdgeInputFormat.class);
  conf.setVertexOutputFormatClass(IdWithValueTextOutputFormat.class);
  Iterable<String> output = InternalVertexRunner.run(conf, new String[0], new String[0]);

  Map<Integer, Double> data = Helpers.parseIntDoubleResults(output);
  assertEquals(3, data.size());
  assertEquals(0.22, data.get(1));
  assertEquals(0.77, data.get(2));
  assertEquals(0.11, data.get(4));
}