Java FileSystem.getLocalFileSystem方法代码示例

import org.apache.flink.core.fs.FileSystem; //导入方法依赖的package包/类
@Test
public void testCompression() throws Exception {
	// given
	final Path outputPath = new Path(File.createTempFile("avro-output-file", "avro").getAbsolutePath());
	final AvroOutputFormat<User> outputFormat = new AvroOutputFormat<>(outputPath, User.class);
	outputFormat.setWriteMode(FileSystem.WriteMode.OVERWRITE);

	final Path compressedOutputPath = new Path(File.createTempFile("avro-output-file", "compressed.avro").getAbsolutePath());
	final AvroOutputFormat<User> compressedOutputFormat = new AvroOutputFormat<>(compressedOutputPath, User.class);
	compressedOutputFormat.setWriteMode(FileSystem.WriteMode.OVERWRITE);
	compressedOutputFormat.setCodec(Codec.SNAPPY);

	// when
	output(outputFormat);
	output(compressedOutputFormat);

	// then
	assertTrue(fileSize(outputPath) > fileSize(compressedOutputPath));

	// cleanup
	FileSystem fs = FileSystem.getLocalFileSystem();
	fs.delete(outputPath, false);
	fs.delete(compressedOutputPath, false);
}
 

import org.apache.flink.core.fs.FileSystem; //导入方法依赖的package包/类
@Test
public void testCompression() throws Exception {
	// given
	final Path outputPath = new Path(File.createTempFile("avro-output-file", "avro").getAbsolutePath());
	final AvroOutputFormat<User> outputFormat = new AvroOutputFormat<>(outputPath, User.class);
	outputFormat.setWriteMode(FileSystem.WriteMode.OVERWRITE);

	final Path compressedOutputPath = new Path(File.createTempFile("avro-output-file", "compressed.avro").getAbsolutePath());
	final AvroOutputFormat<User> compressedOutputFormat = new AvroOutputFormat<>(compressedOutputPath, User.class);
	compressedOutputFormat.setWriteMode(FileSystem.WriteMode.OVERWRITE);
	compressedOutputFormat.setCodec(AvroOutputFormat.Codec.SNAPPY);

	// when
	output(outputFormat);
	output(compressedOutputFormat);

	// then
	assertTrue(fileSize(outputPath) > fileSize(compressedOutputPath));

	// cleanup
	FileSystem fs = FileSystem.getLocalFileSystem();
	fs.delete(outputPath, false);
	fs.delete(compressedOutputPath, false);
}
 

import org.apache.flink.core.fs.FileSystem; //导入方法依赖的package包/类
@Test
public void testGenericRecord() throws IOException {
	final Path outputPath = new Path(File.createTempFile("avro-output-file", "generic.avro").getAbsolutePath());
	final AvroOutputFormat<GenericRecord> outputFormat = new AvroOutputFormat<>(outputPath, GenericRecord.class);
	Schema schema = new Schema.Parser().parse("{\"type\":\"record\", \"name\":\"user\", \"fields\": [{\"name\":\"user_name\", \"type\":\"string\"}, {\"name\":\"favorite_number\", \"type\":\"int\"}, {\"name\":\"favorite_color\", \"type\":\"string\"}]}");
	outputFormat.setWriteMode(FileSystem.WriteMode.OVERWRITE);
	outputFormat.setSchema(schema);
	output(outputFormat, schema);

	GenericDatumReader<GenericRecord> reader = new GenericDatumReader<>(schema);
	DataFileReader<GenericRecord> dataFileReader = new DataFileReader<>(new File(outputPath.getPath()), reader);

	while (dataFileReader.hasNext()) {
		GenericRecord record = dataFileReader.next();
		assertEquals(record.get("user_name").toString(), "testUser");
		assertEquals(record.get("favorite_number"), 1);
		assertEquals(record.get("favorite_color").toString(), "blue");
	}

	//cleanup
	FileSystem fs = FileSystem.getLocalFileSystem();
	fs.delete(outputPath, false);

}
 

import org.apache.flink.core.fs.FileSystem; //导入方法依赖的package包/类
@Test
public void testDeletePathIfEmpty() throws IOException {
	final FileSystem localFs = FileSystem.getLocalFileSystem();

	final File dir = tmp.newFolder();
	assertTrue(dir.exists());

	final Path dirPath = new Path(dir.toURI());

	// deleting an empty directory should work
	assertTrue(FileUtils.deletePathIfEmpty(localFs, dirPath));

	// deleting a non existing directory should work
	assertTrue(FileUtils.deletePathIfEmpty(localFs, dirPath));

	// create a non-empty dir
	final File nonEmptyDir = tmp.newFolder();
	final Path nonEmptyDirPath = new Path(nonEmptyDir.toURI());
	new FileOutputStream(new File(nonEmptyDir, "filename")).close();
	assertFalse(FileUtils.deletePathIfEmpty(localFs, nonEmptyDirPath));
}
 

import org.apache.flink.core.fs.FileSystem; //导入方法依赖的package包/类
@Test
public void testRenameToNonEmptyTargetDir() throws IOException {
	final FileSystem fs = FileSystem.getLocalFileSystem();

	// a source folder with a file
	final File srcFolder = temporaryFolder.newFolder();
	final File srcFile = new File(srcFolder, "someFile.txt");
	assertTrue(srcFile.createNewFile());

	// a non-empty destination folder
	final File dstFolder = temporaryFolder.newFolder();
	final File dstFile  = new File(dstFolder, "target");
	assertTrue(dstFile.createNewFile());

	// this cannot succeed because the destination folder is not empty
	assertFalse(fs.rename(new Path(srcFolder.toURI()), new Path(dstFolder.toURI())));

	// retry after deleting the occupying target file
	assertTrue(dstFile.delete());
	assertTrue(fs.rename(new Path(srcFolder.toURI()), new Path(dstFolder.toURI())));
	assertTrue(new File(dstFolder, srcFile.getName()).exists());
}
 

import org.apache.flink.core.fs.FileSystem; //导入方法依赖的package包/类
/**
 * Validates that even empty streams create a file and a file state handle.
 */
@Test
public void testEmptyState() throws Exception {
	final FileSystem fs = FileSystem.getLocalFileSystem();
	final Path path = new Path(new Path(tmp.newFolder().toURI()), "myFileName");

	final FileStateHandle handle;
	try (FixFileFsStateOutputStream stream = new FixFileFsStateOutputStream(fs, path)) {
		handle = stream.closeAndGetHandle();
	}

	// must have created a handle
	assertNotNull(handle);
	assertEquals(path, handle.getFilePath());

	// the pointer path should exist as a directory
	assertTrue(fs.exists(handle.getFilePath()));
	assertFalse(fs.getFileStatus(path).isDir());

	// the contents should be empty
	try (FSDataInputStream in = handle.openInputStream()) {
		assertEquals(-1, in.read());
	}
}
 

import org.apache.flink.core.fs.FileSystem; //导入方法依赖的package包/类
/**
 * Tests that the underlying stream file is deleted upon calling close.
 */
@Test
public void testCleanupWhenClosingStream() throws IOException {
	final FileSystem fs = FileSystem.getLocalFileSystem();
	final Path path = new Path(new Path(tmp.newFolder().toURI()), "nonCreativeTestFileName");

	// write some test data and close the stream
	try (FixFileFsStateOutputStream stream = new FixFileFsStateOutputStream(fs, path)) {
		Random rnd = new Random();
		for (int i = 0; i < rnd.nextInt(1000); i++) {
			stream.write(rnd.nextInt(100));
		}
		assertTrue(fs.exists(path));
	}

	assertFalse(fs.exists(path));
}
 

import org.apache.flink.core.fs.FileSystem; //导入方法依赖的package包/类
@Test
public void testGetPos() throws Exception {
	FsCheckpointStreamFactory.CheckpointStateOutputStream stream =
			new FsCheckpointStreamFactory.FsCheckpointStateOutputStream(TEMP_DIR_PATH, FileSystem.getLocalFileSystem(), 31, 17);

	for (int i = 0; i < 64; ++i) {
		Assert.assertEquals(i, stream.getPos());
		stream.write(0x42);
	}

	stream.closeAndGetHandle();

	// ----------------------------------------------------

	stream = new FsCheckpointStreamFactory.FsCheckpointStateOutputStream(TEMP_DIR_PATH, FileSystem.getLocalFileSystem(), 31, 17);

	byte[] data = "testme!".getBytes(ConfigConstants.DEFAULT_CHARSET);

	for (int i = 0; i < 7; ++i) {
		Assert.assertEquals(i * (1 + data.length), stream.getPos());
		stream.write(0x42);
		stream.write(data);
	}

	stream.closeAndGetHandle();
}
 

import org.apache.flink.core.fs.FileSystem; //导入方法依赖的package包/类
/**
 * Test that {@link FileUtils#deletePathIfEmpty(FileSystem, Path)} deletes the path if it is
 * empty. A path can only be empty if it is a directory which does not contain any
 * files/directories.
 */
@Test
public void testDeletePathIfEmpty() throws IOException {
	File file = temporaryFolder.newFile();
	File directory = temporaryFolder.newFolder();
	File directoryFile = new File(directory, UUID.randomUUID().toString());

	assertTrue(directoryFile.createNewFile());

	Path filePath = new Path(file.toURI());
	Path directoryPath = new Path(directory.toURI());
	Path directoryFilePath = new Path(directoryFile.toURI());

	FileSystem fs = FileSystem.getLocalFileSystem();

	// verify that the files have been created
	assertTrue(fs.exists(filePath));
	assertTrue(fs.exists(directoryFilePath));

	// delete the single file
	assertFalse(FileUtils.deletePathIfEmpty(fs, filePath));
	assertTrue(fs.exists(filePath));

	// try to delete the non-empty directory
	assertFalse(FileUtils.deletePathIfEmpty(fs, directoryPath));
	assertTrue(fs.exists(directoryPath));

	// delete the file contained in the directory
	assertTrue(fs.delete(directoryFilePath, false));

	// now the deletion should work
	assertTrue(FileUtils.deletePathIfEmpty(fs, directoryPath));
	assertFalse(fs.exists(directoryPath));
}
 

import org.apache.flink.core.fs.FileSystem; //导入方法依赖的package包/类
@Test
public void testRenameNonExistingFile() throws IOException {
	final FileSystem fs = FileSystem.getLocalFileSystem();

	final File srcFile = new File(temporaryFolder.newFolder(), "someFile.txt");
	final File destFile  = new File(temporaryFolder.newFolder(), "target");

	final Path srcFilePath = new Path(srcFile.toURI());
	final Path destFilePath = new Path(destFile.toURI());

	// this cannot succeed because the source file does not exist
	assertFalse(fs.rename(srcFilePath, destFilePath));
}
 

import org.apache.flink.core.fs.FileSystem; //导入方法依赖的package包/类
private static List<String> findTestFiles() throws Exception {
	List<String> files = new ArrayList<>();
	FileSystem fs = FileSystem.getLocalFileSystem();
	FileStatus[] status = fs.listStatus(getBaseTestPythonDir());
	for (FileStatus f : status) {
		String file = f.getPath().toString();
		if (file.endsWith(".py")) {
			files.add(file);
		}
	}
	return files;
}
 

import org.apache.flink.core.fs.FileSystem; //导入方法依赖的package包/类
@Test
public void testEmptyState() throws Exception {
	FsCheckpointStreamFactory.CheckpointStateOutputStream stream =
			new FsCheckpointStreamFactory.FsCheckpointStateOutputStream(TEMP_DIR_PATH, FileSystem.getLocalFileSystem(), 1024, 512);

	StreamStateHandle handle = stream.closeAndGetHandle();
	assertTrue(handle == null);
}
 

import org.apache.flink.core.fs.FileSystem; //导入方法依赖的package包/类
private void runTest(int numBytes, int bufferSize, int threshold, boolean expectFile) throws Exception {
	FsCheckpointStreamFactory.CheckpointStateOutputStream stream =
		new FsCheckpointStreamFactory.FsCheckpointStateOutputStream(
			TEMP_DIR_PATH, FileSystem.getLocalFileSystem(), bufferSize, threshold);

	Random rnd = new Random();
	byte[] original = new byte[numBytes];
	byte[] bytes = new byte[original.length];

	rnd.nextBytes(original);
	System.arraycopy(original, 0, bytes, 0, original.length);

	// the test writes a mixture of writing individual bytes and byte arrays
	int pos = 0;
	while (pos < bytes.length) {
		boolean single = rnd.nextBoolean();
		if (single) {
			stream.write(bytes[pos++]);
		}
		else {
			int num = rnd.nextInt(Math.min(10, bytes.length - pos));
			stream.write(bytes, pos, num);
			pos += num;
		}
	}

	StreamStateHandle handle = stream.closeAndGetHandle();
	if (expectFile) {
		assertTrue(handle instanceof FileStateHandle);
	} else {
		assertTrue(handle instanceof ByteStreamStateHandle);
	}

	// make sure the writing process did not alter the original byte array
	assertArrayEquals(original, bytes);

	try (InputStream inStream = handle.openInputStream()) {
		byte[] validation = new byte[bytes.length];

		DataInputStream dataInputStream = new DataInputStream(inStream);
		dataInputStream.readFully(validation);

		assertArrayEquals(bytes, validation);
	}

	handle.discardState();
}
 

import org.apache.flink.core.fs.FileSystem; //导入方法依赖的package包/类
@Test
public void testRenamePath() throws IOException {
	final File rootDirectory = temporaryFolder.newFolder();

	//create a file /root/src/B/test.csv
	final File srcDirectory  = new File(new File(rootDirectory, "src"), "B");
	assertTrue(srcDirectory.mkdirs());

	final File srcFile = new File(srcDirectory, "test.csv");
	assertTrue(srcFile.createNewFile());

	//Move/rename B and its content to /root/dst/A
	final File destDirectory  = new File(new File(rootDirectory, "dst"), "B");
	final File destFile  = new File(destDirectory, "test.csv");

	final Path srcDirPath = new Path(srcDirectory.toURI());
	final Path srcFilePath = new Path(srcFile.toURI());
	final Path destDirPath = new Path(destDirectory.toURI());
	final Path destFilePath = new Path(destFile.toURI());

	FileSystem fs = FileSystem.getLocalFileSystem();

	// pre-conditions: /root/src/B exists but /root/dst/B does not
	assertTrue(fs.exists(srcDirPath));
	assertFalse(fs.exists(destDirPath));

	// do the move/rename: /root/src/B -> /root/dst/
	assertTrue(fs.rename(srcDirPath, destDirPath));

	// post-conditions: /root/src/B doesn't exists, /root/dst/B/test.csv has been created
	assertTrue(fs.exists(destFilePath));
	assertFalse(fs.exists(srcDirPath));

	// re-create source file and test overwrite
	assertTrue(srcDirectory.mkdirs());
	assertTrue(srcFile.createNewFile());

	// overwrite the destination file
	assertTrue(fs.rename(srcFilePath, destFilePath));

	// post-conditions: now only the src file has been moved
	assertFalse(fs.exists(srcFilePath));
	assertTrue(fs.exists(srcDirPath));
	assertTrue(fs.exists(destFilePath));
}
 

import org.apache.flink.core.fs.FileSystem; //导入方法依赖的package包/类
@Test
public void testKind() {
	final FileSystem fs = FileSystem.getLocalFileSystem();
	assertEquals(FileSystemKind.FILE_SYSTEM, fs.getKind());
}