Java FileSystem.get方法代码示例

import org.apache.flink.core.fs.FileSystem; //导入方法依赖的package包/类
/**
 *  register cache files in program level
 * @param entry contains all relevant information
 * @param name user defined name of that file
 * @throws java.io.IOException
 */
public void registerCachedFile(String name, DistributedCacheEntry entry) throws IOException {
	if (!this.cacheFile.containsKey(name)) {
		try {
			URI u = new URI(entry.filePath);
			if (!u.getPath().startsWith("/")) {
				u = new File(entry.filePath).toURI();
			}
			FileSystem fs = FileSystem.get(u);
			if (fs.exists(new Path(u.getPath()))) {
				this.cacheFile.put(name, new DistributedCacheEntry(u.toString(), entry.isExecutable));
			} else {
				throw new IOException("File " + u.toString() + " doesn't exist.");
			}
		} catch (URISyntaxException ex) {
			throw new IOException("Invalid path: " + entry.filePath, ex);
		}
	} else {
		throw new IOException("cache file " + name + "already exists!");
	}
}
 

import org.apache.flink.core.fs.FileSystem; //导入方法依赖的package包/类
@Override
public void run() {
	try {
		final FileSystem fs = FileSystem.get(this.split.getPath().toUri());
		this.fdis = fs.open(this.split.getPath());
		
		// check for canceling and close the stream in that case, because no one will obtain it
		if (this.aborted) {
			final FSDataInputStream f = this.fdis;
			this.fdis = null;
			f.close();
		}
	}
	catch (Throwable t) {
		this.error = t;
	}
}
 

import org.apache.flink.core.fs.FileSystem; //导入方法依赖的package包/类
@Test
public void testExplicitlySetToOther() throws Exception {
	final Configuration conf = new Configuration();
	conf.setString(CoreOptions.DEFAULT_FILESYSTEM_SCHEME, "otherFS://localhost:1234/");
	FileSystem.initialize(conf);

	URI justPath = new URI(tempFolder.newFile().toURI().getPath());
	assertNull(justPath.getScheme());

	try {
		FileSystem.get(justPath);
		fail("should have failed with an exception");
	}
	catch (UnsupportedFileSystemSchemeException e) {
		assertTrue(e.getMessage().contains("otherFS"));
	}
}
 

import org.apache.flink.core.fs.FileSystem; //导入方法依赖的package包/类
/**
 * Tests that multiple externalized checkpoints can be stored to the same
 * directory.
 */
@Test
public void testStoreExternalizedCheckpointsToSameDirectory() throws Exception {
	String root = tmp.newFolder().getAbsolutePath();
	FileSystem fs = FileSystem.get(new Path(root).toUri());

	// Store
	SavepointV2 savepoint = new SavepointV2(
		1929292,
		CheckpointTestUtils.createOperatorStates(4, 24),
		Collections.<MasterState>emptyList());

	FileStateHandle store1 = SavepointStore.storeExternalizedCheckpointToHandle(root, savepoint);
	fs.exists(store1.getFilePath());
	assertTrue(store1.getFilePath().getPath().contains(SavepointStore.EXTERNALIZED_CHECKPOINT_METADATA_FILE));

	FileStateHandle store2 = SavepointStore.storeExternalizedCheckpointToHandle(root, savepoint);
	fs.exists(store2.getFilePath());
	assertTrue(store2.getFilePath().getPath().contains(SavepointStore.EXTERNALIZED_CHECKPOINT_METADATA_FILE));
}
 

import org.apache.flink.core.fs.FileSystem; //导入方法依赖的package包/类
@BeforeClass
public static void createHDFS() {
	Assume.assumeTrue("HDFS cluster cannot be started on Windows without extensions.", !OperatingSystem.isWindows());

	try {
		Configuration hdConf = new Configuration();
		hdConf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, tempFolder.newFolder().getAbsolutePath());
		MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(hdConf);
		hdfsCluster = builder.build();

		hdfsRootUri = "hdfs://" + hdfsCluster.getURI().getHost() + ":"
				+ hdfsCluster.getNameNodePort() + "/";

		fs = FileSystem.get(new URI(hdfsRootUri));
	}
	catch (Exception e) {
		e.printStackTrace();
		fail("Could not create HDFS mini cluster " + e.getMessage());
	}
}
 

import org.apache.flink.core.fs.FileSystem; //导入方法依赖的package包/类
@Test
public void testConfigPropagation() throws Exception{
	final Configuration conf = new Configuration();
	conf.setString("s3.access-key", "test_access_key_id");
	conf.setString("s3.secret-key", "test_secret_access_key");

	FileSystem.initialize(conf);

	FileSystem fs = FileSystem.get(new URI("s3://test"));
	validateBasicCredentials(fs);
}
 

import org.apache.flink.core.fs.FileSystem; //导入方法依赖的package包/类
@Test
public void testConfigPropagationWithPrestoPrefix() throws Exception{
	final Configuration conf = new Configuration();
	conf.setString("presto.s3.access-key", "test_access_key_id");
	conf.setString("presto.s3.secret-key", "test_secret_access_key");

	FileSystem.initialize(conf);

	FileSystem fs = FileSystem.get(new URI("s3://test"));
	validateBasicCredentials(fs);
}
 

import org.apache.flink.core.fs.FileSystem; //导入方法依赖的package包/类
@Test
public void testConfigPropagationAlternateStyle() throws Exception{
	final Configuration conf = new Configuration();
	conf.setString("s3.access.key", "test_access_key_id");
	conf.setString("s3.secret.key", "test_secret_access_key");

	FileSystem.initialize(conf);

	FileSystem fs = FileSystem.get(new URI("s3://test"));
	validateBasicCredentials(fs);
}
 

import org.apache.flink.core.fs.FileSystem; //导入方法依赖的package包/类
private FileSystem getFileSystem() throws IOException {
    if (fileSystem == null) {
        try {
            fileSystem = FileSystem.get(new URI(inputPath));
        } catch (URISyntaxException e) {
            throw new RuntimeException(e);
        }
    }
    return fileSystem;
}
 

import org.apache.flink.core.fs.FileSystem; //导入方法依赖的package包/类
public InputStream openFile(Path path) throws IOException {
		FileSystem fs = FileSystem.get(path.toUri());
		return fs.open(path);
}
 

import org.apache.flink.core.fs.FileSystem; //导入方法依赖的package包/类
/**
 * Obtains basic file statistics containing only file size. If the input is a directory, then the size is the sum of all contained files.
 * 
 * @see org.apache.flink.api.common.io.InputFormat#getStatistics(org.apache.flink.api.common.io.statistics.BaseStatistics)
 */
@Override
public FileBaseStatistics getStatistics(BaseStatistics cachedStats) throws IOException {
	
	final FileBaseStatistics cachedFileStats = cachedStats instanceof FileBaseStatistics ?
		(FileBaseStatistics) cachedStats : null;
			
	try {
		final Path path = this.filePath;
		final FileSystem fs = FileSystem.get(path.toUri());
		
		return getFileStats(cachedFileStats, path, fs, new ArrayList<FileStatus>(1));
	} catch (IOException ioex) {
		if (LOG.isWarnEnabled()) {
			LOG.warn("Could not determine statistics for file '" + this.filePath + "' due to an io error: "
					+ ioex.getMessage());
		}
	}
	catch (Throwable t) {
		if (LOG.isErrorEnabled()) {
			LOG.error("Unexpected problem while getting the file statistics for file '" + this.filePath + "': "
					+ t.getMessage(), t);
		}
	}
	
	// no statistics available
	return null;
}
 

import org.apache.flink.core.fs.FileSystem; //导入方法依赖的package包/类
@Test
public void testDefaultsToLocal() throws Exception {
	URI justPath = new URI(tempFolder.newFile().toURI().getPath());
	assertNull(justPath.getScheme());

	FileSystem fs = FileSystem.get(justPath);
	assertEquals("file", fs.getUri().getScheme());
}
 

import org.apache.flink.core.fs.FileSystem; //导入方法依赖的package包/类
@Override
public void run(SourceFunction.SourceContext<TimestampedFileInputSplit> context) throws Exception {
	Path p = new Path(path);
	FileSystem fileSystem = FileSystem.get(p.toUri());
	if (!fileSystem.exists(p)) {
		throw new FileNotFoundException("The provided file path " + path + " does not exist.");
	}

	checkpointLock = context.getCheckpointLock();
	switch (watchType) {
		case PROCESS_CONTINUOUSLY:
			while (isRunning) {
				synchronized (checkpointLock) {
					monitorDirAndForwardSplits(fileSystem, context);
				}
				Thread.sleep(interval);
			}

			// here we do not need to set the running to false and the
			// globalModificationTime to Long.MAX_VALUE because to arrive here,
			// either close() or cancel() have already been called, so this
			// is already done.

			break;
		case PROCESS_ONCE:
			synchronized (checkpointLock) {

				// the following check guarantees that if we restart
				// after a failure and we managed to have a successful
				// checkpoint, we will not reprocess the directory.

				if (globalModificationTime == Long.MIN_VALUE) {
					monitorDirAndForwardSplits(fileSystem, context);
					globalModificationTime = Long.MAX_VALUE;
				}
				isRunning = false;
			}
			break;
		default:
			isRunning = false;
			throw new RuntimeException("Unknown WatchType" + watchType);
	}
}
 

import org.apache.flink.core.fs.FileSystem; //导入方法依赖的package包/类
@Test
public void testConfiguration() throws Exception {

	// nothing configured, we should get a regular file system
	FileSystem hdfs = FileSystem.get(URI.create("hdfs://localhost:12345/a/b/c"));
	FileSystem ftpfs = FileSystem.get(URI.create("ftp://localhost:12345/a/b/c"));

	assertFalse(hdfs instanceof LimitedConnectionsFileSystem);
	assertFalse(ftpfs instanceof LimitedConnectionsFileSystem);

	// configure some limits, which should cause "fsScheme" to be limited

	final Configuration config = new Configuration();
	config.setInteger("fs.hdfs.limit.total", 40);
	config.setInteger("fs.hdfs.limit.input", 39);
	config.setInteger("fs.hdfs.limit.output", 38);
	config.setInteger("fs.hdfs.limit.timeout", 23456);
	config.setInteger("fs.hdfs.limit.stream-timeout", 34567);

	try {
		FileSystem.initialize(config);

		hdfs = FileSystem.get(URI.create("hdfs://localhost:12345/a/b/c"));
		ftpfs = FileSystem.get(URI.create("ftp://localhost:12345/a/b/c"));

		assertTrue(hdfs instanceof LimitedConnectionsFileSystem);
		assertFalse(ftpfs instanceof LimitedConnectionsFileSystem);

		LimitedConnectionsFileSystem limitedFs = (LimitedConnectionsFileSystem) hdfs;
		assertEquals(40, limitedFs.getMaxNumOpenStreamsTotal());
		assertEquals(39, limitedFs.getMaxNumOpenInputStreams());
		assertEquals(38, limitedFs.getMaxNumOpenOutputStreams());
		assertEquals(23456, limitedFs.getStreamOpenTimeout());
		assertEquals(34567, limitedFs.getStreamInactivityTimeout());
	}
	finally {
		// clear all settings
		FileSystem.initialize(new Configuration());
	}
}
 

import org.apache.flink.core.fs.FileSystem; //导入方法依赖的package包/类
/**
 * Gets the file system that stores the file state.
 * @return The file system that stores the file state.
 * @throws IOException Thrown if the file system cannot be accessed.
 */
protected FileSystem getFileSystem() throws IOException {
	if (fs == null) {
		fs = FileSystem.get(filePath.toUri());
	}
	return fs;
}