Java FileSystem.initialize方法代码示例

import org.apache.flink.core.fs.FileSystem; //导入方法依赖的package包/类
@Test
public void testExplicitlySetToOther() throws Exception {
	final Configuration conf = new Configuration();
	conf.setString(CoreOptions.DEFAULT_FILESYSTEM_SCHEME, "otherFS://localhost:1234/");
	FileSystem.initialize(conf);

	URI justPath = new URI(tempFolder.newFile().toURI().getPath());
	assertNull(justPath.getScheme());

	try {
		FileSystem.get(justPath);
		fail("should have failed with an exception");
	}
	catch (UnsupportedFileSystemSchemeException e) {
		assertTrue(e.getMessage().contains("otherFS"));
	}
}
 

import org.apache.flink.core.fs.FileSystem; //导入方法依赖的package包/类
public CliFrontend(String configDir) throws Exception {
	configurationDirectory = Preconditions.checkNotNull(configDir);

	// configure the config directory
	File configDirectory = new File(configDir);
	LOG.info("Using configuration directory " + configDirectory.getAbsolutePath());

	// load the configuration
	LOG.info("Trying to load configuration file");
	this.config = GlobalConfiguration.loadConfiguration(configDirectory.getAbsolutePath());

	try {
		FileSystem.initialize(config);
	} catch (IOException e) {
		throw new Exception("Error while setting the default " +
			"filesystem scheme from configuration.", e);
	}

	this.clientTimeout = AkkaUtils.getClientTimeout(config);
	this.defaultParallelism = GlobalConfiguration.loadConfiguration().getInteger(
													ConfigConstants.DEFAULT_PARALLELISM_KEY,
													ConfigConstants.DEFAULT_PARALLELISM);
}
 

import org.apache.flink.core.fs.FileSystem; //导入方法依赖的package包/类
public CliFrontend(
		Configuration configuration,
		List<CustomCommandLine<?>> customCommandLines) throws Exception {
	this.configuration = Preconditions.checkNotNull(configuration);
	this.customCommandLines = Preconditions.checkNotNull(customCommandLines);

	try {
		FileSystem.initialize(this.configuration);
	} catch (IOException e) {
		throw new Exception("Error while setting the default " +
			"filesystem scheme from configuration.", e);
	}

	this.customCommandLineOptions = new Options();

	for (CustomCommandLine<?> customCommandLine : customCommandLines) {
		customCommandLine.addGeneralOptions(customCommandLineOptions);
		customCommandLine.addRunOptions(customCommandLineOptions);
	}

	this.clientTimeout = AkkaUtils.getClientTimeout(this.configuration);
	this.defaultParallelism = configuration.getInteger(
		ConfigConstants.DEFAULT_PARALLELISM_KEY,
		ConfigConstants.DEFAULT_PARALLELISM);
}
 

import org.apache.flink.core.fs.FileSystem; //导入方法依赖的package包/类
@BeforeClass
public static void checkCredentialsAndSetup() throws IOException {
	// check whether credentials exist
	Assume.assumeTrue("AWS S3 bucket not configured, skipping test...", BUCKET != null);
	Assume.assumeTrue("AWS S3 access key not configured, skipping test...", ACCESS_KEY != null);
	Assume.assumeTrue("AWS S3 secret key not configured, skipping test...", SECRET_KEY != null);

	// initialize configuration with valid credentials
	final Configuration conf = new Configuration();
	conf.setString("s3.access.key", ACCESS_KEY);
	conf.setString("s3.secret.key", SECRET_KEY);
	FileSystem.initialize(conf);

	// check for uniqueness of the test directory
	final Path directory = new Path("s3://" + BUCKET + '/' + TEST_DATA_DIR);
	final FileSystem fs = directory.getFileSystem();

	// directory must not yet exist
	assertFalse(fs.exists(directory));

	// reset configuration
	FileSystem.initialize(new Configuration());

	skipTest = false;
}
 

import org.apache.flink.core.fs.FileSystem; //导入方法依赖的package包/类
@AfterClass
public static void cleanUp() throws IOException {
	if (!skipTest) {
		// initialize configuration with valid credentials
		final Configuration conf = new Configuration();
		conf.setString("s3.access.key", ACCESS_KEY);
		conf.setString("s3.secret.key", SECRET_KEY);
		FileSystem.initialize(conf);

		final Path directory = new Path("s3://" + BUCKET + '/' + TEST_DATA_DIR);
		final FileSystem fs = directory.getFileSystem();

		// clean up
		fs.delete(directory, true);

		// now directory must be gone
		assertFalse(fs.exists(directory));

		// reset configuration
		FileSystem.initialize(new Configuration());
	}
}
 

import org.apache.flink.core.fs.FileSystem; //导入方法依赖的package包/类
@Test
public void testDefaultFsParameterLoading() throws Exception {
	try {
		final File tmpDir = temporaryFolder.newFolder();
		final File confFile = new File(tmpDir, GlobalConfiguration.FLINK_CONF_FILENAME);

		final URI defaultFS = new URI("otherFS", null, "localhost", 1234, null, null, null);

		final PrintWriter pw1 = new PrintWriter(confFile);
		pw1.println("fs.default-scheme: " + defaultFS);
		pw1.close();

		String[] args = new String[] {"--configDir:" + tmpDir};
		TaskManager.parseArgsAndLoadConfig(args);

		assertEquals(defaultFS, FileSystem.getDefaultFsUri());
	}
	finally {
		// reset FS settings
		FileSystem.initialize(new Configuration());
	}
}
 

import org.apache.flink.core.fs.FileSystem; //导入方法依赖的package包/类
@Test
public void testExplicitlySetToLocal() throws Exception {
	final Configuration conf = new Configuration();
	conf.setString(CoreOptions.DEFAULT_FILESYSTEM_SCHEME, LocalFileSystem.getLocalFsURI().toString());
	FileSystem.initialize(conf);

	URI justPath = new URI(tempFolder.newFile().toURI().getPath());
	assertNull(justPath.getScheme());

	FileSystem fs = FileSystem.get(justPath);
	assertEquals("file", fs.getUri().getScheme());
}
 

import org.apache.flink.core.fs.FileSystem; //导入方法依赖的package包/类
@BeforeClass
public static void checkCredentialsAndSetup() throws IOException {
	// check whether credentials exist
	Assume.assumeTrue("AWS S3 bucket not configured, skipping test...", BUCKET != null);
	Assume.assumeTrue("AWS S3 access key not configured, skipping test...", ACCESS_KEY != null);
	Assume.assumeTrue("AWS S3 secret key not configured, skipping test...", SECRET_KEY != null);

	// initialize configuration with valid credentials
	final Configuration conf = new Configuration();
	conf.setString("s3.access.key", ACCESS_KEY);
	conf.setString("s3.secret.key", SECRET_KEY);
	FileSystem.initialize(conf);
}
 

import org.apache.flink.core.fs.FileSystem; //导入方法依赖的package包/类
@Test
public void testSimpleFileWriteAndRead() throws Exception {
	final Configuration conf = new Configuration();
	conf.setString("s3.access.key", ACCESS_KEY);
	conf.setString("s3.secret.key", SECRET_KEY);

	final String testLine = "Hello Upload!";

	FileSystem.initialize(conf);

	final Path path = new Path("s3://" + BUCKET + '/' + TEST_DATA_DIR + "/test.txt");
	final FileSystem fs = path.getFileSystem();

	try {
		try (FSDataOutputStream out = fs.create(path, WriteMode.OVERWRITE);
				OutputStreamWriter writer = new OutputStreamWriter(out, StandardCharsets.UTF_8)) {
			writer.write(testLine);
		}

		try (FSDataInputStream in = fs.open(path);
				InputStreamReader ir = new InputStreamReader(in, StandardCharsets.UTF_8);
				BufferedReader reader = new BufferedReader(ir)) {
			String line = reader.readLine();
			assertEquals(testLine, line);
		}
	}
	finally {
		fs.delete(path, false);
	}
}
 

import org.apache.flink.core.fs.FileSystem; //导入方法依赖的package包/类
protected void configureFileSystems(Configuration configuration) throws Exception {
	LOG.info("Install default filesystem.");

	try {
		FileSystem.initialize(configuration);
	} catch (IOException e) {
		throw new IOException("Error while setting the default " +
			"filesystem scheme from configuration.", e);
	}
}
 

import org.apache.flink.core.fs.FileSystem; //导入方法依赖的package包/类
@Test
public void testConfigPropagation() throws Exception{
	final Configuration conf = new Configuration();
	conf.setString("s3.access-key", "test_access_key_id");
	conf.setString("s3.secret-key", "test_secret_access_key");

	FileSystem.initialize(conf);

	FileSystem fs = FileSystem.get(new URI("s3://test"));
	validateBasicCredentials(fs);
}
 

import org.apache.flink.core.fs.FileSystem; //导入方法依赖的package包/类
@Test
public void testConfigPropagationWithPrestoPrefix() throws Exception{
	final Configuration conf = new Configuration();
	conf.setString("presto.s3.access-key", "test_access_key_id");
	conf.setString("presto.s3.secret-key", "test_secret_access_key");

	FileSystem.initialize(conf);

	FileSystem fs = FileSystem.get(new URI("s3://test"));
	validateBasicCredentials(fs);
}
 

import org.apache.flink.core.fs.FileSystem; //导入方法依赖的package包/类
@Test
public void testConfigPropagationAlternateStyle() throws Exception{
	final Configuration conf = new Configuration();
	conf.setString("s3.access.key", "test_access_key_id");
	conf.setString("s3.secret.key", "test_secret_access_key");

	FileSystem.initialize(conf);

	FileSystem fs = FileSystem.get(new URI("s3://test"));
	validateBasicCredentials(fs);
}
 

import org.apache.flink.core.fs.FileSystem; //导入方法依赖的package包/类
/**
 * The instance entry point for the Mesos AppMaster. Obtains user group
 * information and calls the main work method {@link #runPrivileged(Configuration, Configuration)} as a
 * privileged action.
 *
 * @param args The command line arguments.
 * @return The process exit code.
 */
protected int run(final String[] args) {
	try {
		LOG.debug("All environment variables: {}", ENV);

		// loading all config values here has the advantage that the program fails fast, if any
		// configuration problem occurs

		CommandLineParser parser = new PosixParser();
		CommandLine cmd = parser.parse(ALL_OPTIONS, args);

		final Configuration dynamicProperties = BootstrapTools.parseDynamicProperties(cmd);
		final Configuration config = MesosEntrypointUtils.loadConfiguration(dynamicProperties, LOG);

		// configure the filesystems
		try {
			FileSystem.initialize(config);
		} catch (IOException e) {
			throw new IOException("Error while configuring the filesystems.", e);
		}

		// configure security
		SecurityConfiguration sc = new SecurityConfiguration(config);
		SecurityUtils.install(sc);

		// run the actual work in the installed security context
		return SecurityUtils.getInstalledContext().runSecured(new Callable<Integer>() {
			@Override
			public Integer call() throws Exception {
				return runPrivileged(config, dynamicProperties);
			}
		});
	}
	catch (Throwable t) {
		// make sure that everything whatever ends up in the log
		LOG.error("Mesos AppMaster initialization failed", t);
		return INIT_ERROR_EXIT_CODE;
	}
}
 

import org.apache.flink.core.fs.FileSystem; //导入方法依赖的package包/类
@After
public void clearFsSettings() throws IOException {
	FileSystem.initialize(new Configuration());
}