Java DataStatistics类代码示例

import org.apache.flink.optimizer.DataStatistics; //导入依赖的package包/类
/**
 * Creates a instance that submits the programs to the JobManager defined in the
 * configuration. This method will try to resolve the JobManager hostname and throw an exception
 * if that is not possible.
 *
 * @param flinkConfig The config used to obtain the job-manager's address, and used to configure the optimizer.
 * @param highAvailabilityServices HighAvailabilityServices to use for leader retrieval
 */
public ClusterClient(Configuration flinkConfig, HighAvailabilityServices highAvailabilityServices) {
	this.flinkConfig = Preconditions.checkNotNull(flinkConfig);
	this.compiler = new Optimizer(new DataStatistics(), new DefaultCostEstimator(), flinkConfig);

	this.timeout = AkkaUtils.getClientTimeout(flinkConfig);
	this.lookupTimeout = AkkaUtils.getLookupTimeout(flinkConfig);

	this.actorSystemLoader = new LazyActorSystemLoader(
		highAvailabilityServices,
		Time.milliseconds(lookupTimeout.toMillis()),
		flinkConfig,
		log);

	this.highAvailabilityServices = Preconditions.checkNotNull(highAvailabilityServices);
}
 

import org.apache.flink.optimizer.DataStatistics; //导入依赖的package包/类
/**
 * The default estimates build on the principle of inclusion: The smaller input key domain is included in the larger
 * input key domain. We also assume that every key from the larger input has one join partner in the smaller input.
 * The result cardinality is hence the larger one.
 */
@Override
protected void computeOperatorSpecificDefaultEstimates(DataStatistics statistics) {
	long card1 = getFirstPredecessorNode().getEstimatedNumRecords();
	long card2 = getSecondPredecessorNode().getEstimatedNumRecords();
	this.estimatedNumRecords = (card1 < 0 || card2 < 0) ? -1 : Math.max(card1, card2);
	
	if (this.estimatedNumRecords >= 0) {
		float width1 = getFirstPredecessorNode().getEstimatedAvgWidthPerOutputRecord();
		float width2 = getSecondPredecessorNode().getEstimatedAvgWidthPerOutputRecord();
		float width = (width1 <= 0 || width2 <= 0) ? -1 : width1 + width2;
		
		if (width > 0) {
			this.estimatedOutputSize = (long) (width * this.estimatedNumRecords);
		}
	}
}
 

import org.apache.flink.optimizer.DataStatistics; //导入依赖的package包/类
@Override
protected void computeOperatorSpecificDefaultEstimates(DataStatistics statistics) {
	long card1 = getFirstPredecessorNode().getEstimatedNumRecords();
	long card2 = getSecondPredecessorNode().getEstimatedNumRecords();

	if (card1 < 0 || card2 < 0) {
		this.estimatedNumRecords = -1;
	} else {
		this.estimatedNumRecords = Math.max(card1, card2);
	}

	if (this.estimatedNumRecords >= 0) {
		float width1 = getFirstPredecessorNode().getEstimatedAvgWidthPerOutputRecord();
		float width2 = getSecondPredecessorNode().getEstimatedAvgWidthPerOutputRecord();
		float width = (width1 <= 0 || width2 <= 0) ? -1 : width1 + width2;

		if (width > 0) {
			this.estimatedOutputSize = (long) (width * this.estimatedNumRecords);
		}
	}
}
 

import org.apache.flink.optimizer.DataStatistics; //导入依赖的package包/类
/**
 * We assume that the cardinality is the product of  the input cardinalities
 * and that the result width is the sum of the input widths.
 * 
 * @param statistics The statistics object to optionally access.
 */
@Override
protected void computeOperatorSpecificDefaultEstimates(DataStatistics statistics) {
	long card1 = getFirstPredecessorNode().getEstimatedNumRecords();
	long card2 = getSecondPredecessorNode().getEstimatedNumRecords();
	this.estimatedNumRecords = (card1 < 0 || card2 < 0) ? -1 : card1 * card2;
	
	if (this.estimatedNumRecords >= 0) {
		float width1 = getFirstPredecessorNode().getEstimatedAvgWidthPerOutputRecord();
		float width2 = getSecondPredecessorNode().getEstimatedAvgWidthPerOutputRecord();
		float width = (width1 <= 0 || width2 <= 0) ? -1 : width1 + width2;
		
		if (width > 0) {
			this.estimatedOutputSize = (long) (width * this.estimatedNumRecords);
		}
	}
}
 

import org.apache.flink.optimizer.DataStatistics; //导入依赖的package包/类
/**
 * Helpers to generate the JobGraph.
 */
private static JobGraph getOptimizedPlan(Plan plan) {
	Optimizer pc = new Optimizer(new DataStatistics(), new Configuration());
	JobGraphGenerator jgg = new JobGraphGenerator();
	OptimizedPlan op = pc.compile(plan);
	return jgg.compileJobGraph(op);
}
 

import org.apache.flink.optimizer.DataStatistics; //导入依赖的package包/类
protected Tuple2<JobGraph, ClassLoader> getJobGraphAndClassLoader(JarActionHandlerConfig config) throws Exception {
	// generate the graph
	JobGraph graph = null;

	PackagedProgram program = new PackagedProgram(
			new File(jarDir, config.getJarFile()),
			config.getEntryClass(),
			config.getProgramArgs());
	ClassLoader classLoader = program.getUserCodeClassLoader();

	Optimizer optimizer = new Optimizer(new DataStatistics(), new DefaultCostEstimator(), new Configuration());
	FlinkPlan plan = ClusterClient.getOptimizedPlan(optimizer, program, config.getParallelism());

	if (plan instanceof StreamingPlan) {
		graph = ((StreamingPlan) plan).getJobGraph();
	} else if (plan instanceof OptimizedPlan) {
		graph = new JobGraphGenerator().compileJobGraph((OptimizedPlan) plan);
	}
	if (graph == null) {
		throw new CompilerException("A valid job graph couldn't be generated for the jar.");
	}

	// Set the savepoint settings
	graph.setSavepointRestoreSettings(config.getSavepointRestoreSettings());

	for (URL jar : program.getAllLibraries()) {
		try {
			graph.addJar(new Path(jar.toURI()));
		}
		catch (URISyntaxException e) {
			throw new ProgramInvocationException("Invalid jar path. Unexpected error. :(");
		}
	}
	return Tuple2.of(graph, classLoader);
}
 

import org.apache.flink.optimizer.DataStatistics; //导入依赖的package包/类
/**
 * Creates a JSON representation of the given dataflow's execution plan.
 *
 * @param plan The dataflow plan.
 * @return The dataflow's execution plan, as a JSON string.
 * @throws Exception Thrown, if the optimization process that creates the execution plan failed.
 */
@Override
public String getOptimizerPlanAsJSON(Plan plan) throws Exception {
	final int parallelism = plan.getDefaultParallelism() == ExecutionConfig.PARALLELISM_DEFAULT ? 1 : plan.getDefaultParallelism();

	Optimizer pc = new Optimizer(new DataStatistics(), this.configuration);
	pc.setDefaultParallelism(parallelism);
	OptimizedPlan op = pc.compile(plan);

	return new PlanJSONDumpGenerator().getOptimizerPlanAsJSON(op);
}
 

import org.apache.flink.optimizer.DataStatistics; //导入依赖的package包/类
/**
 * Creates a JSON representation of the given dataflow's execution plan.
 *
 * @param plan The dataflow plan.
 * @return The dataflow's execution plan, as a JSON string.
 * @throws Exception Thrown, if the optimization process that creates the execution plan failed.
 */
public static String optimizerPlanAsJSON(Plan plan) throws Exception {
	final int parallelism = plan.getDefaultParallelism() == ExecutionConfig.PARALLELISM_DEFAULT ? 1 : plan.getDefaultParallelism();

	Optimizer pc = new Optimizer(new DataStatistics(), new Configuration());
	pc.setDefaultParallelism(parallelism);
	OptimizedPlan op = pc.compile(plan);

	return new PlanJSONDumpGenerator().getOptimizerPlanAsJSON(op);
}
 

import org.apache.flink.optimizer.DataStatistics; //导入依赖的package包/类
@Test
public void testGetExecutionPlan() {
	try {
		jobManagerSystem.actorOf(
			Props.create(FailureReturningActor.class),
			JobMaster.JOB_MANAGER_NAME);

		PackagedProgram prg = new PackagedProgram(TestOptimizerPlan.class, "/dev/random", "/tmp");
		assertNotNull(prg.getPreviewPlan());

		Optimizer optimizer = new Optimizer(new DataStatistics(), new DefaultCostEstimator(), config);
		OptimizedPlan op = (OptimizedPlan) ClusterClient.getOptimizedPlan(optimizer, prg, 1);
		assertNotNull(op);

		PlanJSONDumpGenerator dumper = new PlanJSONDumpGenerator();
		assertNotNull(dumper.getOptimizerPlanAsJSON(op));

		// test HTML escaping
		PlanJSONDumpGenerator dumper2 = new PlanJSONDumpGenerator();
		dumper2.setEncodeForHTML(true);
		String htmlEscaped = dumper2.getOptimizerPlanAsJSON(op);

		assertEquals(-1, htmlEscaped.indexOf('\\'));
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 

import org.apache.flink.optimizer.DataStatistics; //导入依赖的package包/类
@Test
public void testGetExecutionPlan() {
	try {
		PackagedProgram prg = new PackagedProgram(TestOptimizerPlan.class, "/dev/random", "/tmp");
		assertNotNull(prg.getPreviewPlan());

		InetAddress mockAddress = InetAddress.getLocalHost();
		InetSocketAddress mockJmAddress = new InetSocketAddress(mockAddress, 12345);

		Configuration config = new Configuration();

		config.setString(JobManagerOptions.ADDRESS, mockJmAddress.getHostName());
		config.setInteger(JobManagerOptions.PORT, mockJmAddress.getPort());

		Optimizer optimizer = new Optimizer(new DataStatistics(), new DefaultCostEstimator(), config);
		OptimizedPlan op = (OptimizedPlan) ClusterClient.getOptimizedPlan(optimizer, prg, -1);
		assertNotNull(op);

		PlanJSONDumpGenerator dumper = new PlanJSONDumpGenerator();
		assertNotNull(dumper.getOptimizerPlanAsJSON(op));

		// test HTML escaping
		PlanJSONDumpGenerator dumper2 = new PlanJSONDumpGenerator();
		dumper2.setEncodeForHTML(true);
		String htmlEscaped = dumper2.getOptimizerPlanAsJSON(op);

		assertEquals(-1, htmlEscaped.indexOf('\\'));
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 

import org.apache.flink.optimizer.DataStatistics; //导入依赖的package包/类
@Override
protected void computeOperatorSpecificDefaultEstimates(DataStatistics statistics) {
	long card1 = getFirstPredecessorNode().getEstimatedNumRecords();
	long card2 = getSecondPredecessorNode().getEstimatedNumRecords();
	this.estimatedNumRecords = (card1 < 0 || card2 < 0) ? -1 : card1 + card2;
	
	long size1 = getFirstPredecessorNode().getEstimatedOutputSize();
	long size2 = getSecondPredecessorNode().getEstimatedOutputSize();
	this.estimatedOutputSize = (size1 < 0 || size2 < 0) ? -1 : size1 + size2;
}
 

import org.apache.flink.optimizer.DataStatistics; //导入依赖的package包/类
@Override
public void computeOutputEstimates(DataStatistics statistics) {
	OptimizerNode in1 = getFirstPredecessorNode();
	OptimizerNode in2 = getSecondPredecessorNode();
	
	this.estimatedNumRecords = in1.estimatedNumRecords > 0 && in2.estimatedNumRecords > 0 ?
			in1.estimatedNumRecords + in2.estimatedNumRecords : -1;
	this.estimatedOutputSize = in1.estimatedOutputSize > 0 && in2.estimatedOutputSize > 0 ?
		in1.estimatedOutputSize + in2.estimatedOutputSize : -1;
}
 

import org.apache.flink.optimizer.DataStatistics; //导入依赖的package包/类
@Before
public void setup() {
	Configuration flinkConf = new Configuration();
	this.dataStats = new DataStatistics();
	this.withStatsCompiler = new Optimizer(this.dataStats, new DefaultCostEstimator(), flinkConf);
	this.withStatsCompiler.setDefaultParallelism(DEFAULT_PARALLELISM);

	this.noStatsCompiler = new Optimizer(null, new DefaultCostEstimator(), flinkConf);
	this.noStatsCompiler.setDefaultParallelism(DEFAULT_PARALLELISM);
}
 

import org.apache.flink.optimizer.DataStatistics; //导入依赖的package包/类
private OptimizedPlan compileProgram(String jobName) {
	Plan p = createProgramPlan(jobName);

	Optimizer pc = new Optimizer(new DataStatistics(), new Configuration());
	return pc.compile(p);
}
 

import org.apache.flink.optimizer.DataStatistics; //导入依赖的package包/类
private JobGraph getJobGraph(final Plan plan) throws Exception {
	final Optimizer pc = new Optimizer(new DataStatistics(), this.executor.configuration());
	final OptimizedPlan op = pc.compile(plan);
	final JobGraphGenerator jgg = new JobGraphGenerator();
	return jgg.compileJobGraph(op);
}