Java PageRankData类代码示例

import org.apache.flink.examples.java.graph.util.PageRankData; //导入依赖的package包/类
private static DataSet<Long> getPagesDataSet(ExecutionEnvironment env, ParameterTool params) {
	if (params.has("pages")) {
		return env.readCsvFile(params.get("pages"))
			.fieldDelimiter(" ")
			.lineDelimiter("\n")
			.types(Long.class)
			.map(new MapFunction<Tuple1<Long>, Long>() {
				@Override
				public Long map(Tuple1<Long> v) {
					return v.f0;
				}
			});
	} else {
		System.out.println("Executing PageRank example with default pages data set.");
		System.out.println("Use --pages to specify file input.");
		return PageRankData.getDefaultPagesDataSet(env);
	}
}
 

import org.apache.flink.examples.java.graph.util.PageRankData; //导入依赖的package包/类
private static boolean parseParameters(String[] args) {
	
	if(args.length > 0) {
		if(args.length == 5) {
			fileOutput = true;
			pagesInputPath = args[0];
			linksInputPath = args[1];
			outputPath = args[2];
			numPages = Integer.parseInt(args[3]);
			maxIterations = Integer.parseInt(args[4]);
		} else {
			System.err.println("Usage: PageRankBasic <pages path> <links path> <output path> <num pages> <num iterations>");
			return false;
		}
	} else {
		System.out.println("Executing PageRank Basic example with default parameters and built-in default data.");
		System.out.println("  Provide parameters to read input data from files.");
		System.out.println("  See the documentation for the correct format of input files.");
		System.out.println("  Usage: PageRankBasic <pages path> <links path> <output path> <num pages> <num iterations>");
		
		numPages = PageRankData.getNumberOfPages();
	}
	return true;
}
 

import org.apache.flink.examples.java.graph.util.PageRankData; //导入依赖的package包/类
private static DataSet<Tuple2<Long, Long>> getLinksDataSet(ExecutionEnvironment env, ParameterTool params) {
	if (params.has("links")) {
		return env.readCsvFile(params.get("links"))
			.fieldDelimiter(" ")
			.lineDelimiter("\n")
			.types(Long.class, Long.class);
	} else {
		System.out.println("Executing PageRank example with default links data set.");
		System.out.println("Use --links to specify file input.");
		return PageRankData.getDefaultEdgeDataSet(env);
	}
}
 

import org.apache.flink.examples.java.graph.util.PageRankData; //导入依赖的package包/类
private static DataSet<Long> getPagesDataSet(ExecutionEnvironment env) {
	if(fileOutput) {
		return env
					.readCsvFile(pagesInputPath)
						.fieldDelimiter(' ')
						.lineDelimiter("\n")
						.types(Long.class)
					.map(new MapFunction<Tuple1<Long>, Long>() {
						@Override
						public Long map(Tuple1<Long> v) { return v.f0; }
					});
	} else {
		return PageRankData.getDefaultPagesDataSet(env);
	}
}
 

import org.apache.flink.examples.java.graph.util.PageRankData; //导入依赖的package包/类
private static DataSet<Tuple2<Long, Long>> getLinksDataSet(ExecutionEnvironment env) {
	if(fileOutput) {
		return env.readCsvFile(linksInputPath)
					.fieldDelimiter(' ')
					.lineDelimiter("\n")
					.types(Long.class, Long.class);
	} else {
		return PageRankData.getDefaultEdgeDataSet(env);
	}
}
 

import org.apache.flink.examples.java.graph.util.PageRankData; //导入依赖的package包/类
public static void main(String[] args) throws Exception {

		ParameterTool params = ParameterTool.fromArgs(args);

		final int numPages = params.getInt("numPages", PageRankData.getNumberOfPages());
		final int maxIterations = params.getInt("iterations", 10);

		// set up execution environment
		final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

		// make the parameters available to the web ui
		env.getConfig().setGlobalJobParameters(params);

		// get input data
		DataSet<Long> pagesInput = getPagesDataSet(env, params);
		DataSet<Tuple2<Long, Long>> linksInput = getLinksDataSet(env, params);

		// assign initial rank to pages
		DataSet<Tuple2<Long, Double>> pagesWithRanks = pagesInput.
				map(new RankAssigner((1.0d / numPages)));

		// build adjacency list from link input
		DataSet<Tuple2<Long, Long[]>> adjacencyListInput =
				linksInput.groupBy(0).reduceGroup(new BuildOutgoingEdgeList());

		// set iterative data set
		IterativeDataSet<Tuple2<Long, Double>> iteration = pagesWithRanks.iterate(maxIterations);

		DataSet<Tuple2<Long, Double>> newRanks = iteration
				// join pages with outgoing edges and distribute rank
				.join(adjacencyListInput).where(0).equalTo(0).flatMap(new JoinVertexWithEdgesMatch())
				// collect and sum ranks
				.groupBy(0).aggregate(SUM, 1)
				// apply dampening factor
				.map(new Dampener(DAMPENING_FACTOR, numPages));

		DataSet<Tuple2<Long, Double>> finalPageRanks = iteration.closeWith(
				newRanks,
				newRanks.join(iteration).where(0).equalTo(0)
				// termination condition
				.filter(new EpsilonFilter()));

		// emit result
		if (params.has("output")) {
			finalPageRanks.writeAsCsv(params.get("output"), "\n", " ");
			// execute program
			env.execute("Basic Page Rank Example");
		} else {
			System.out.println("Printing result to stdout. Use --output to specify output path.");
			finalPageRanks.print();
		}
	}