Java Configuration類代碼示例

import org.apache.flink.configuration.Configuration; //導入依賴的package包/類
@Override
public void open(Configuration config) {
  ValueStateDescriptor<AbstractStatisticsWrapper<AisMessage>> descriptor =
      new ValueStateDescriptor<AbstractStatisticsWrapper<AisMessage>>("trajectoryStatistics",
          TypeInformation.of(new TypeHint<AbstractStatisticsWrapper<AisMessage>>() {}));

  statisticsOfTrajectory = getRuntimeContext().getState(descriptor);

}
 

import org.apache.flink.configuration.Configuration; //導入依賴的package包/類
@AfterClass
public static void cleanUp() throws IOException {
	if (!skipTest) {
		// initialize configuration with valid credentials
		final Configuration conf = new Configuration();
		conf.setString("s3.access.key", ACCESS_KEY);
		conf.setString("s3.secret.key", SECRET_KEY);
		FileSystem.initialize(conf);

		final Path directory = new Path("s3://" + BUCKET + '/' + TEST_DATA_DIR);
		final FileSystem fs = directory.getFileSystem();

		// clean up
		fs.delete(directory, true);

		// now directory must be gone
		assertFalse(fs.exists(directory));

		// reset configuration
		FileSystem.initialize(new Configuration());
	}
}
 

import org.apache.flink.configuration.Configuration; //導入依賴的package包/類
@Override
public YarnClusterClient deploy() {
  ApplicationSubmissionContext context = Records.newRecord(ApplicationSubmissionContext.class);
  context.setApplicationId(job.yarnAppId());
  ApplicationReport report;
  try {
    report = startAppMaster(context);

    Configuration conf = getFlinkConfiguration();
    conf.setString(JobManagerOptions.ADDRESS.key(), report.getHost());
    conf.setInteger(JobManagerOptions.PORT.key(), report.getRpcPort());

    return createYarnClusterClient(this, yarnClient, report, conf, false);
  } catch (Exception e) {
    throw new RuntimeException(e);
  }
}
 

import org.apache.flink.configuration.Configuration; //導入依賴的package包/類
@Test
public void testDeployerWithIsolatedConfiguration() throws Exception {
  YarnClusterConfiguration clusterConf = mock(YarnClusterConfiguration.class);
  doReturn(new YarnConfiguration()).when(clusterConf).conf();
  ScheduledExecutorService executor = mock(ScheduledExecutorService.class);
  Configuration flinkConf = new Configuration();
  YarnClient client = mock(YarnClient.class);
  JobDeployer deploy = new JobDeployer(clusterConf, client, executor, flinkConf);
  AthenaXYarnClusterDescriptor desc = mock(AthenaXYarnClusterDescriptor.class);

  YarnClusterClient clusterClient = mock(YarnClusterClient.class);
  doReturn(clusterClient).when(desc).deploy();

  ActorGateway actorGateway = mock(ActorGateway.class);
  doReturn(actorGateway).when(clusterClient).getJobManagerGateway();
  doReturn(Future$.MODULE$.successful(null)).when(actorGateway).ask(any(), any());

  JobGraph jobGraph = mock(JobGraph.class);
  doReturn(JobID.generate()).when(jobGraph).getJobID();
  deploy.start(desc, jobGraph);

  ArgumentCaptor<Configuration> args = ArgumentCaptor.forClass(Configuration.class);
  verify(desc).setFlinkConfiguration(args.capture());
  assertNotSame(flinkConf, args.getValue());
}
 

import org.apache.flink.configuration.Configuration; //導入依賴的package包/類
@Override
public void open(Configuration parameters) throws Exception {
    super.open(parameters);

    state = new State<>();

    processingTimeService =
            ((StreamingRuntimeContext) getRuntimeContext()).getProcessingTimeService();

    long currentProcessingTime = processingTimeService.getCurrentProcessingTime();

    processingTimeService.registerTimer(currentProcessingTime + inactiveBucketCheckInterval, this);

    this.clock = new Clock() {
        @Override
        public long currentTimeMillis() {
            return processingTimeService.getCurrentProcessingTime();
        }
    };
}
 

import org.apache.flink.configuration.Configuration; //導入依賴的package包/類
/**
 * Create a file system with the user-defined {@code HDFS} configuration.
 *
 * @throws IOException
 */
private void initFileSystem() throws IOException {
    if (fs != null) {
        return;
    }
    org.apache.hadoop.conf.Configuration hadoopConf = HadoopFileSystem.getHadoopConfiguration();
    if (fsConfig != null) {
        String disableCacheName = String.format("fs.%s.impl.disable.cache", new Path(basePath).toUri().getScheme());
        hadoopConf.setBoolean(disableCacheName, true);
        for (String key : fsConfig.keySet()) {
            hadoopConf.set(key, fsConfig.getString(key, null));
        }
    }

    fs = new Path(basePath).getFileSystem(hadoopConf);
}
 

import org.apache.flink.configuration.Configuration; //導入依賴的package包/類
/**
 * 通用入HDFS-Map
 * @param topic topic名
 * @return SaveHdfsSink
 */
public static SinkFunction<String> getSink(String topic) {
    Configuration configuration = new Configuration();
    //TODO 添加Hadoop配置內容
    configuration.setString("dfs.namenode.name.dir", "file:///home/hadmin/data/hadoop/hdfs/name");
    configuration.setString("dfs.nameservices", "ns");
    configuration.setString("dfs.ha.namenodes.ns", "nn1,nn2");
    configuration.setString("dfs.namenode.rpc-address.ns.nn1", "10.11.0.193:9000");
    configuration.setString("dfs.namenode.rpc-address.ns.nn2", "10.11.0.194:9000");
    configuration.setString("dfs.namenode.shared.edits.dir", "qjournal://10.11.0.193:8485;10.11.0.194;10.11.0.195:8485/ns");
    configuration.setString("hadoop.tmp.dir", "/home/hadmin/data/hadoop/tmp");
    configuration.setString("fs.defaultFS", "hdfs://ns");
    configuration.setString("dfs.journalnode.edits.dir", "/home/hadmin/data/hadoop/journal");
    configuration.setString("ha.zookeeper.quorum", "10.11.0.193:2181,10.11.0.194:2181,10.11.0.195:2181");
    configuration.setString("mapreduce.input.fileinputformat.split.minsize", "10");

    TODBucketingSink<String> sink = new TODBucketingSink<>("/xml/" + topic + "/");
    sink.setBucketer(new DateTimeBucketer<>("yyyy/MM/dd/HH"));
    sink.setWriter(new StringWriter<>());
    sink.setPendingPrefix("source");
    sink.setPendingSuffix(".txt");
    sink.setFSConfig(configuration);
    //設置Flink Bucketer 的刷新時間
    sink.setAsyncTimeout(60000L);
    return sink;
}
 

import org.apache.flink.configuration.Configuration; //導入依賴的package包/類
/**
 * Override open from RichFunctions set
 * This method allow the retrieval of the state and set it as queryable
 */
@Override
@SuppressWarnings("unchecked")
public void open(Configuration config) {
    ValueStateDescriptor descriptor =
            new ValueStateDescriptor(
                    "consumption",
                    LampConsumption.class); // default value of the state, if nothing was set
    this.consumption = getRuntimeContext().getState(descriptor);
    descriptor.setQueryable("consumption-list-api");
}
 

import org.apache.flink.configuration.Configuration; //導入依賴的package包/類
public static void setup(int proxyPortRangeStart, int serverPortRangeStart) {
	try {
		Configuration config = new Configuration();
		config.setLong(TaskManagerOptions.MANAGED_MEMORY_SIZE, 4L);
		config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, NUM_TMS);
		config.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, NUM_SLOTS_PER_TM);
		config.setInteger(QueryableStateOptions.CLIENT_NETWORK_THREADS, 1);
		config.setBoolean(QueryableStateOptions.SERVER_ENABLE, true);
		config.setInteger(QueryableStateOptions.SERVER_NETWORK_THREADS, 1);
		config.setString(QueryableStateOptions.PROXY_PORT_RANGE, proxyPortRangeStart + "-" + (proxyPortRangeStart + NUM_TMS));
		config.setString(QueryableStateOptions.SERVER_PORT_RANGE, serverPortRangeStart + "-" + (serverPortRangeStart + NUM_TMS));

		cluster = new TestingCluster(config, false);
		cluster.start(true);

		client = new QueryableStateClient("localhost", proxyPortRangeStart);

		// verify that we are not in HA mode
		Assert.assertTrue(cluster.haMode() == HighAvailabilityMode.NONE);

	} catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 

import org.apache.flink.configuration.Configuration; //導入依賴的package包/類
@Before
public void createHDFS() {
	try {
		baseDir = new File("./target/hdfs/hdfsTesting").getAbsoluteFile();
		FileUtil.fullyDelete(baseDir);

		org.apache.hadoop.conf.Configuration hdConf = new org.apache.hadoop.conf.Configuration();
		hdConf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, baseDir.getAbsolutePath());
		hdConf.set("dfs.block.size", String.valueOf(1048576)); // this is the minimum we can set.

		MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(hdConf);
		hdfsCluster = builder.build();

		hdfsURI = "hdfs://" + hdfsCluster.getURI().getHost() + ":" + hdfsCluster.getNameNodePort() + "/";
		hdfs = new org.apache.hadoop.fs.Path(hdfsURI).getFileSystem(hdConf);

	} catch (Throwable e) {
		e.printStackTrace();
		Assert.fail("Test failed " + e.getMessage());
	}
}
 

import org.apache.flink.configuration.Configuration; //導入依賴的package包/類
@Test
public void testPrintSinkStdErr() throws Exception {
	ByteArrayOutputStream baos = new ByteArrayOutputStream();
	PrintStream stream = new PrintStream(baos);
	System.setOut(stream);

	final StreamingRuntimeContext ctx = Mockito.mock(StreamingRuntimeContext.class);

	PrintSinkFunction<String> printSink = new PrintSinkFunction<>();
	printSink.setRuntimeContext(ctx);
	try {
		printSink.open(new Configuration());
	} catch (Exception e) {
		Assert.fail();
	}
	printSink.setTargetToStandardErr();
	printSink.invoke("hello world!", SinkContextUtil.forTimestamp(0));

	assertEquals("Print to System.err", printSink.toString());
	assertEquals("hello world!" + line, baos.toString());

	printSink.close();
	stream.close();
}
 

import org.apache.flink.configuration.Configuration; //導入依賴的package包/類
@Test
public void testOperatorNameTruncation() {
	Configuration cfg = new Configuration();
	cfg.setString(MetricOptions.SCOPE_NAMING_OPERATOR, ScopeFormat.SCOPE_OPERATOR_NAME);
	MetricRegistryImpl registry = new MetricRegistryImpl(MetricRegistryConfiguration.fromConfiguration(cfg));
	TaskManagerMetricGroup tm = new TaskManagerMetricGroup(registry, "host", "id");
	TaskManagerJobMetricGroup job = new TaskManagerJobMetricGroup(registry, tm, new JobID(), "jobname");
	TaskMetricGroup taskMetricGroup = new TaskMetricGroup(registry, job, new JobVertexID(), new AbstractID(), "task", 0, 0);

	String originalName = new String(new char[100]).replace("\0", "-");
	OperatorMetricGroup operatorMetricGroup = taskMetricGroup.addOperator(originalName);

	String storedName = operatorMetricGroup.getScopeComponents()[0];
	Assert.assertEquals(TaskMetricGroup.METRICS_OPERATOR_NAME_MAX_LENGTH, storedName.length());
	Assert.assertEquals(originalName.substring(0, TaskMetricGroup.METRICS_OPERATOR_NAME_MAX_LENGTH), storedName);
}
 

import org.apache.flink.configuration.Configuration; //導入依賴的package包/類
/**
 * Verifies that {@link TaskManagerServicesConfiguration#hasNewNetworkBufConf(Configuration)}
 * returns the correct result for mixed old/new configurations.
 */
@SuppressWarnings("deprecation")
@Test
public void hasNewNetworkBufConfMixed() throws Exception {
	Configuration config = new Configuration();
	assertTrue(TaskManagerServicesConfiguration.hasNewNetworkBufConf(config));

	config.setInteger(TaskManagerOptions.NETWORK_NUM_BUFFERS, 1);
	assertFalse(TaskManagerServicesConfiguration.hasNewNetworkBufConf(config));

	// old + 1 new parameter = new:
	Configuration config1 = config.clone();
	config1.setFloat(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_FRACTION, 0.1f);
	assertTrue(TaskManagerServicesConfiguration.hasNewNetworkBufConf(config1));

	config1 = config.clone();
	config1.setLong(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MIN, 1024);
	assertTrue(TaskManagerServicesConfiguration.hasNewNetworkBufConf(config1));

	config1 = config.clone();
	config1.setLong(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MAX, 1024);
	assertTrue(TaskManagerServicesConfiguration.hasNewNetworkBufConf(config1));
}
 

import org.apache.flink.configuration.Configuration; //導入依賴的package包/類
public MesosFlinkResourceManager(
	Configuration flinkConfig,
	MesosConfiguration mesosConfig,
	MesosWorkerStore workerStore,
	LeaderRetrievalService leaderRetrievalService,
	MesosTaskManagerParameters taskManagerParameters,
	ContainerSpecification taskManagerContainerSpec,
	MesosArtifactResolver artifactResolver,
	int maxFailedTasks,
	int numInitialTaskManagers) {

	super(numInitialTaskManagers, flinkConfig, leaderRetrievalService);

	this.mesosConfig = requireNonNull(mesosConfig);

	this.workerStore = requireNonNull(workerStore);
	this.artifactResolver = requireNonNull(artifactResolver);

	this.taskManagerParameters = requireNonNull(taskManagerParameters);
	this.taskManagerContainerSpec = requireNonNull(taskManagerContainerSpec);
	this.maxFailedTasks = maxFailedTasks;

	this.workersInNew = new HashMap<>();
	this.workersInLaunch = new HashMap<>();
	this.workersBeingReturned = new HashMap<>();
}
 

import org.apache.flink.configuration.Configuration; //導入依賴的package包/類
/**
 * Validates taking the application-defined memory state backend and adding additional
 * parameters from the cluster configuration, but giving precedence to application-defined
 * parameters over configuration-defined parameters.
 */
@Test
public void testConfigureMemoryStateBackendMixed() throws Exception {
	final String appCheckpointDir = new Path(tmp.newFolder().toURI()).toString();
	final String checkpointDir = new Path(tmp.newFolder().toURI()).toString();
	final String savepointDir = new Path(tmp.newFolder().toURI()).toString();

	final Path expectedCheckpointPath = new Path(appCheckpointDir);
	final Path expectedSavepointPath = new Path(savepointDir);

	final MemoryStateBackend backend = new MemoryStateBackend(appCheckpointDir, null);

	final Configuration config = new Configuration();
	config.setString(backendKey, "filesystem"); // check that this is not accidentally picked up
	config.setString(CheckpointingOptions.CHECKPOINTS_DIRECTORY, checkpointDir); // this parameter should not be picked up
	config.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY, savepointDir);

	StateBackend loadedBackend = StateBackendLoader.fromApplicationOrConfigOrDefault(backend, config, cl, null);
	assertTrue(loadedBackend instanceof MemoryStateBackend);

	final MemoryStateBackend memBackend = (MemoryStateBackend) loadedBackend;
	assertEquals(expectedCheckpointPath, memBackend.getCheckpointPath());
	assertEquals(expectedSavepointPath, memBackend.getSavepointPath());
}