Java Tuple3ToEdgeMap类代码示例

import org.apache.flink.graph.utils.Tuple3ToEdgeMap; //导入依赖的package包/类
private static DataSet<Edge<Long, Double>> getEdgesDataSet(ExecutionEnvironment env) {
	if (fileOutput) {
		return env.readCsvFile(edgesInputPath)
				.lineDelimiter("\n")
				.fieldDelimiter("\t")
				.types(Long.class, Long.class, Double.class)
				.map(new Tuple3ToEdgeMap<>());
	} else {
		return SingleSourceShortestPathsData.getDefaultEdgeDataSet(env);
	}
}
 

import org.apache.flink.graph.utils.Tuple3ToEdgeMap; //导入依赖的package包/类
private static DataSet<Edge<Long, Double>> getEdgeDataSet(ExecutionEnvironment env) {
	if (fileOutput) {
		return env.readCsvFile(edgesInputPath)
				.fieldDelimiter("\t")
				.lineDelimiter("\n")
				.types(Long.class, Long.class, Double.class)
				.map(new Tuple3ToEdgeMap<>());
	} else {
		return SingleSourceShortestPathsData.getDefaultEdgeDataSet(env);
	}
}
 

import org.apache.flink.graph.utils.Tuple3ToEdgeMap; //导入依赖的package包/类
private static DataSet<Edge<Long, Double>> getEdgesDataSet(ExecutionEnvironment env) {
	if (fileOutput) {
		return env.readCsvFile(edgesInputPath)
				.lineDelimiter("\n")
				.fieldDelimiter("\t")
				.ignoreComments("%")
				.types(Long.class, Long.class, Double.class)
				.map(new Tuple3ToEdgeMap<>());
	} else {
		return SingleSourceShortestPathsData.getDefaultEdgeDataSet(env);
	}
}
 

import org.apache.flink.graph.utils.Tuple3ToEdgeMap; //导入依赖的package包/类
@Test
public void testTranslation() {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<Long> bcGather = env.fromElements(1L);
	DataSet<Long> bcSum = env.fromElements(1L);
	DataSet<Long> bcApply = env.fromElements(1L);

	DataSet<Vertex<Long, Long>> result;

	// ------------ construct the test program ------------------

	DataSet<Edge<Long, NullValue>> edges = env.fromElements(new Tuple3<>(
		1L, 2L, NullValue.getInstance())).map(new Tuple3ToEdgeMap<>());

	Graph<Long, Long, NullValue> graph = Graph.fromDataSet(edges, new InitVertices(), env);

	GSAConfiguration parameters = new GSAConfiguration();

	parameters.registerAggregator(AGGREGATOR_NAME, new LongSumAggregator());
	parameters.setName(ITERATION_NAME);
	parameters.setParallelism(ITERATION_parallelism);
	parameters.addBroadcastSetForGatherFunction(BC_SET_GATHER_NAME, bcGather);
	parameters.addBroadcastSetForSumFunction(BC_SET_SUM_NAME, bcSum);
	parameters.addBroadcastSetForApplyFunction(BC_SET_APLLY_NAME, bcApply);

	result = graph.runGatherSumApplyIteration(
		new GatherNeighborIds(), new SelectMinId(),
		new UpdateComponentId(), NUM_ITERATIONS, parameters).getVertices();

	result.output(new DiscardingOutputFormat<>());

	// ------------- validate the java program ----------------

	assertTrue(result instanceof DeltaIterationResultSet);

	DeltaIterationResultSet<?, ?> resultSet = (DeltaIterationResultSet<?, ?>) result;
	DeltaIteration<?, ?> iteration = resultSet.getIterationHead();

	// check the basic iteration properties
	assertEquals(NUM_ITERATIONS, resultSet.getMaxIterations());
	assertArrayEquals(new int[]{0}, resultSet.getKeyPositions());
	assertEquals(ITERATION_parallelism, iteration.getParallelism());
	assertEquals(ITERATION_NAME, iteration.getName());

	assertEquals(AGGREGATOR_NAME, iteration.getAggregators().getAllRegisteredAggregators().iterator().next().getName());

	// validate that the semantic properties are set as they should
	TwoInputUdfOperator<?, ?, ?, ?> solutionSetJoin = (TwoInputUdfOperator<?, ?, ?, ?>) resultSet.getNextWorkset();
	assertTrue(solutionSetJoin.getSemanticProperties().getForwardingTargetFields(0, 0).contains(0));
	assertTrue(solutionSetJoin.getSemanticProperties().getForwardingTargetFields(1, 0).contains(0));

	SingleInputUdfOperator<?, ?, ?> sumReduce = (SingleInputUdfOperator<?, ?, ?>) solutionSetJoin.getInput1();
	SingleInputUdfOperator<?, ?, ?> gatherMap = (SingleInputUdfOperator<?, ?, ?>) sumReduce.getInput();

	// validate that the broadcast sets are forwarded
	assertEquals(bcGather, gatherMap.getBroadcastSets().get(BC_SET_GATHER_NAME));
	assertEquals(bcSum, sumReduce.getBroadcastSets().get(BC_SET_SUM_NAME));
	assertEquals(bcApply, solutionSetJoin.getBroadcastSets().get(BC_SET_APLLY_NAME));
}
 

import org.apache.flink.graph.utils.Tuple3ToEdgeMap; //导入依赖的package包/类
@Test
public void testGSACompiler() {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	env.setParallelism(DEFAULT_PARALLELISM);

	// compose test program

	DataSet<Edge<Long, NullValue>> edges = env.fromElements(new Tuple3<>(
		1L, 2L, NullValue.getInstance())).map(new Tuple3ToEdgeMap<>());

	Graph<Long, Long, NullValue> graph = Graph.fromDataSet(edges, new InitVertices(), env);

	DataSet<Vertex<Long, Long>> result = graph.runGatherSumApplyIteration(
		new GatherNeighborIds(), new SelectMinId(),
		new UpdateComponentId(), 100).getVertices();

	result.output(new DiscardingOutputFormat<>());

	Plan p = env.createProgramPlan("GSA Connected Components");
	OptimizedPlan op = compileNoStats(p);

	// check the sink
	SinkPlanNode sink = op.getDataSinks().iterator().next();
	assertEquals(ShipStrategyType.FORWARD, sink.getInput().getShipStrategy());
	assertEquals(DEFAULT_PARALLELISM, sink.getParallelism());
	assertEquals(PartitioningProperty.HASH_PARTITIONED, sink.getGlobalProperties().getPartitioning());

	// check the iteration
	WorksetIterationPlanNode iteration = (WorksetIterationPlanNode) sink.getInput().getSource();
	assertEquals(DEFAULT_PARALLELISM, iteration.getParallelism());

	// check the solution set join and the delta
	PlanNode ssDelta = iteration.getSolutionSetDeltaPlanNode();
	assertTrue(ssDelta instanceof DualInputPlanNode); // this is only true if the update function preserves the partitioning

	DualInputPlanNode ssJoin = (DualInputPlanNode) ssDelta;
	assertEquals(DEFAULT_PARALLELISM, ssJoin.getParallelism());
	assertEquals(ShipStrategyType.PARTITION_HASH, ssJoin.getInput1().getShipStrategy());
	assertEquals(new FieldList(0), ssJoin.getInput1().getShipStrategyKeys());

	// check the workset set join
	SingleInputPlanNode sumReducer = (SingleInputPlanNode) ssJoin.getInput1().getSource();
	SingleInputPlanNode gatherMapper = (SingleInputPlanNode) sumReducer.getInput().getSource();
	DualInputPlanNode edgeJoin = (DualInputPlanNode) gatherMapper.getInput().getSource();
	assertEquals(DEFAULT_PARALLELISM, edgeJoin.getParallelism());
	// input1 is the workset
	assertEquals(ShipStrategyType.FORWARD, edgeJoin.getInput1().getShipStrategy());
	// input2 is the edges
	assertEquals(ShipStrategyType.PARTITION_HASH, edgeJoin.getInput2().getShipStrategy());
	assertTrue(edgeJoin.getInput2().getTempMode().isCached());

	assertEquals(new FieldList(0), edgeJoin.getInput2().getShipStrategyKeys());
}
 

import org.apache.flink.graph.utils.Tuple3ToEdgeMap; //导入依赖的package包/类
@SuppressWarnings("serial")
public static void main(String[] args) throws Exception {

	String edgeInputPath;
	int maxIterations;
	String outputPath;

	if (args.length == 3) {
		edgeInputPath = args[0];
		outputPath = args[1];
		maxIterations = Integer.parseInt(args[2]);

	} else {
		System.err.println("Usage: <input edges path> <output path> <num iterations>");
		return;
	}

	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	//read the Edge DataSet from the input file
	DataSet<Edge<String, Double>> links = env.readCsvFile(edgeInputPath)
			.fieldDelimiter("\t")
			.lineDelimiter("\n")
			.types(String.class, String.class, Double.class)
			.map(new Tuple3ToEdgeMap<String, Double>());
	

	//create a Graph with vertex values initialized to 1.0
	Graph<String, Double, Double> network = Graph.fromDataSet(links,
			new MapFunction<String, Double>() {
				public Double map(String value) throws Exception {
					return 1.0;
				}
			}, env);
	
	
	//for each vertex calculate the total weight of its outgoing edges
	DataSet<Tuple2<String, Double>> sumEdgeWeights = 
			network.reduceOnEdges(new SumWeight(), EdgeDirection.OUT);
				

	// assign the transition probabilities as edge weights:
	//divide edge weight by the total weight of outgoing edges for that source 
	Graph<String, Double, Double> networkWithWeights = network
			.joinWithEdgesOnSource(sumEdgeWeights,
					new EdgeJoinFunction<Double, Double>() {
						@Override
						public Double edgeJoin(Double v1, Double v2) throws Exception {
							return v1 / v2;
						}
					});

	//Now run the Page Rank algorithm over the weighted graph
	DataSet<Vertex<String, Double>> pageRanks = networkWithWeights.run(
			new PageRank<String>(DAMPENING_FACTOR, maxIterations));

	pageRanks.writeAsCsv(outputPath, "\n", "\t");
	// since file sinks are lazy,trigger the execution explicitly
	env.execute("PageRank with Edge Weights");

}
 

import org.apache.flink.graph.utils.Tuple3ToEdgeMap; //导入依赖的package包/类
/**
 * Creates a graph from a DataSet of Tuple2 objects for vertices and
 * Tuple3 objects for edges.
 *
 * <p>The first field of the Tuple2 vertex object will become the vertex ID
 * and the second field will become the vertex value.
 * The first field of the Tuple3 object for edges will become the source ID,
 * the second field will become the target ID, and the third field will become
 * the edge value.
 *
 * @param vertices a DataSet of Tuple2 representing the vertices.
 * @param edges a DataSet of Tuple3 representing the edges.
 * @param context the flink execution environment.
 * @return the newly created graph.
 */
public static <K, VV, EV> Graph<K, VV, EV> fromTupleDataSet(DataSet<Tuple2<K, VV>> vertices,
		DataSet<Tuple3<K, K, EV>> edges, ExecutionEnvironment context) {

	DataSet<Vertex<K, VV>> vertexDataSet = vertices
		.map(new Tuple2ToVertexMap<>())
			.name("Type conversion");

	DataSet<Edge<K, EV>> edgeDataSet = edges
		.map(new Tuple3ToEdgeMap<>())
			.name("Type conversion");

	return fromDataSet(vertexDataSet, edgeDataSet, context);
}