Java FSDataOutputStream类代码示例

import org.apache.flink.core.fs.FSDataOutputStream; //导入依赖的package包/类
private void readStateData(
	Path restoreFilePath,
	StreamStateHandle remoteFileHandle) throws IOException {

	FileSystem restoreFileSystem = restoreFilePath.getFileSystem();

	FSDataInputStream inputStream = null;
	FSDataOutputStream outputStream = null;

	try {
		inputStream = remoteFileHandle.openInputStream();
		stateBackend.cancelStreamRegistry.registerCloseable(inputStream);

		outputStream = restoreFileSystem.create(restoreFilePath, FileSystem.WriteMode.OVERWRITE);
		stateBackend.cancelStreamRegistry.registerCloseable(outputStream);

		byte[] buffer = new byte[8 * 1024];
		while (true) {
			int numBytes = inputStream.read(buffer);
			if (numBytes == -1) {
				break;
			}

			outputStream.write(buffer, 0, numBytes);
		}
	} finally {
		if (inputStream != null && stateBackend.cancelStreamRegistry.unregisterCloseable(inputStream)) {
			inputStream.close();
		}

		if (outputStream != null && stateBackend.cancelStreamRegistry.unregisterCloseable(outputStream)) {
			outputStream.close();
		}
	}
}
 

import org.apache.flink.core.fs.FSDataOutputStream; //导入依赖的package包/类
@Override
public FSDataOutputStream create(final Path filePath, final WriteMode overwrite) throws IOException {
	checkNotNull(filePath, "filePath");

	if (exists(filePath) && overwrite == WriteMode.NO_OVERWRITE) {
		throw new FileAlreadyExistsException("File already exists: " + filePath);
	}

	final Path parent = filePath.getParent();
	if (parent != null && !mkdirs(parent)) {
		throw new IOException("Mkdirs failed to create " + parent);
	}

	final File file = pathToFile(filePath);
	return new LocalDataOutputStream(file);
}
 

import org.apache.flink.core.fs.FSDataOutputStream; //导入依赖的package包/类
@Test
public void testSimpleFileWriteAndRead() throws Exception {
	final Configuration conf = new Configuration();
	conf.setString("s3.access.key", ACCESS_KEY);
	conf.setString("s3.secret.key", SECRET_KEY);

	final String testLine = "Hello Upload!";

	FileSystem.initialize(conf);

	final Path path = new Path("s3://" + BUCKET + '/' + TEST_DATA_DIR + "/test.txt");
	final FileSystem fs = path.getFileSystem();

	try {
		try (FSDataOutputStream out = fs.create(path, WriteMode.OVERWRITE);
				OutputStreamWriter writer = new OutputStreamWriter(out, StandardCharsets.UTF_8)) {
			writer.write(testLine);
		}

		try (FSDataInputStream in = fs.open(path);
				InputStreamReader ir = new InputStreamReader(in, StandardCharsets.UTF_8);
				BufferedReader reader = new BufferedReader(ir)) {
			String line = reader.readLine();
			assertEquals(testLine, line);
		}
	}
	finally {
		fs.delete(path, false);
	}
}
 

import org.apache.flink.core.fs.FSDataOutputStream; //导入依赖的package包/类
@Override
public RetrievableStateHandle<T> store(T state) throws Exception {
	Exception latestException = null;

	for (int attempt = 0; attempt < 10; attempt++) {
		Path filePath = getNewFilePath();

		try (FSDataOutputStream outStream = fs.create(filePath, FileSystem.WriteMode.NO_OVERWRITE)) {
			InstantiationUtil.serializeObject(outStream, state);
			return new RetrievableStreamStateHandle<T>(filePath, outStream.getPos());
		}
		catch (Exception e) {
			latestException = e;
		}
	}

	throw new Exception("Could not open output stream for state backend", latestException);
}
 

import org.apache.flink.core.fs.FSDataOutputStream; //导入依赖的package包/类
public static void copy(Path sourcePath, Path targetPath, boolean executable) throws IOException {
	// TODO rewrite this to make it participate in the closable registry and the lifecycle of a task.
	// we unwrap the file system to get raw streams without safety net
	FileSystem sFS = FileSystem.getUnguardedFileSystem(sourcePath.toUri());
	FileSystem tFS = FileSystem.getUnguardedFileSystem(targetPath.toUri());
	if (!tFS.exists(targetPath)) {
		if (sFS.getFileStatus(sourcePath).isDir()) {
			tFS.mkdirs(targetPath);
			FileStatus[] contents = sFS.listStatus(sourcePath);
			for (FileStatus content : contents) {
				String distPath = content.getPath().toString();
				if (content.isDir()) {
					if (distPath.endsWith("/")) {
						distPath = distPath.substring(0, distPath.length() - 1);
					}
				}
				String localPath = targetPath.toString() + distPath.substring(distPath.lastIndexOf("/"));
				copy(content.getPath(), new Path(localPath), executable);
			}
		} else {
			try (FSDataOutputStream lfsOutput = tFS.create(targetPath, FileSystem.WriteMode.NO_OVERWRITE); FSDataInputStream fsInput = sFS.open(sourcePath)) {
				IOUtils.copyBytes(fsInput, lfsOutput);
				//noinspection ResultOfMethodCallIgnored
				new File(targetPath.toString()).setExecutable(executable);
			} catch (IOException ioe) {
				LOG.error("could not copy file to local file cache.", ioe);
			}
		}
	}
}
 

import org.apache.flink.core.fs.FSDataOutputStream; //导入依赖的package包/类
/**
 * Tests loading with unexpected magic number.
 */
@Test
public void testUnexpectedSavepoint() throws Exception {
	FsSavepointStore store = new FsSavepointStore(tmp.getRoot().getPath(), "fs-savepoint-store-test-");

	// Random file
	Path filePath = new Path(tmp.getRoot().getPath(), UUID.randomUUID().toString());
	FSDataOutputStream fdos = FileSystem.get(filePath.toUri()).create(filePath, false);
	DataOutputStream dos = new DataOutputStream(fdos);
	for (int i = 0; i < 10; i++) {
		dos.writeLong(ThreadLocalRandom.current().nextLong());
	}

	try {
		store.loadSavepoint(filePath.toString());
		fail("Did not throw expected Exception");
	} catch (RuntimeException e) {
		assertTrue(e.getMessage().contains("Flink 1.0") && e.getMessage().contains("Unexpected magic number"));
	}
}
 

import org.apache.flink.core.fs.FSDataOutputStream; //导入依赖的package包/类
/**
 * Tests loading with unexpected magic number.
 */
@Test
public void testUnexpectedSavepoint() throws Exception {
	// Random file
	Path filePath = new Path(tmp.getRoot().getPath(), UUID.randomUUID().toString());
	FSDataOutputStream fdos = FileSystem.get(filePath.toUri()).create(filePath, FileSystem.WriteMode.NO_OVERWRITE);
	DataOutputStream dos = new DataOutputStream(fdos);
	for (int i = 0; i < 10; i++) {
		dos.writeLong(ThreadLocalRandom.current().nextLong());
	}

	try {
		SavepointStore.loadSavepoint(filePath.toString(), Thread.currentThread().getContextClassLoader());
		fail("Did not throw expected Exception");
	} catch (RuntimeException e) {
		assertTrue(e.getMessage().contains("Flink 1.0") && e.getMessage().contains("Unexpected magic number"));
	}
}
 

import org.apache.flink.core.fs.FSDataOutputStream; //导入依赖的package包/类
/**
 * Tests that the underlying stream file is deleted upon calling close.
 */
@Test
public void testCleanupWhenClosingStream() throws IOException {

	final FileSystem fs = mock(FileSystem.class);
	final FSDataOutputStream outputStream = mock(FSDataOutputStream.class);

	final ArgumentCaptor<Path> pathCaptor = ArgumentCaptor.forClass(Path.class);

	when(fs.create(pathCaptor.capture(), any(FileSystem.WriteMode.class))).thenReturn(outputStream);

	CheckpointStreamFactory.CheckpointStateOutputStream stream = new FsCheckpointStreamFactory.FsCheckpointStateOutputStream(
		TEMP_DIR_PATH,
		fs,
		4,
		0);

	// this should create the underlying file stream
	stream.write(new byte[]{1,2,3,4,5});

	verify(fs).create(any(Path.class), any(FileSystem.WriteMode.class));

	stream.close();

	verify(fs).delete(eq(pathCaptor.getValue()), anyBoolean());
}
 

import org.apache.flink.core.fs.FSDataOutputStream; //导入依赖的package包/类
@Override
public FSDataOutputStream create(final Path f, final boolean overwrite, final int bufferSize,
		final short replication, final long blockSize) throws IOException {

	if (exists(f) && !overwrite) {
		throw new IOException("File already exists:" + f);
	}

	final Path parent = f.getParent();
	if (parent != null && !mkdirs(parent)) {
		throw new IOException("Mkdirs failed to create " + parent.toString());
	}

	final File file = pathToFile(f);
	return new LocalDataOutputStream(file);
}
 

import org.apache.flink.core.fs.FSDataOutputStream; //导入依赖的package包/类
@Override
public FSDataOutputStream create(final Path f, final boolean overwrite, final int bufferSize,
		final short replication, final long blockSize)
		throws IOException {

	if (!overwrite && exists(f)) {
		throw new IOException(f.toUri() + " already exists");
	}

	final S3BucketObjectPair bop = this.directoryStructure.toBucketObjectPair(f);
	if (!bop.hasBucket() || !bop.hasObject()) {
		throw new IOException(f.toUri() + " is not a valid path to create a new file");
	}

	if (bufferSize < S3DataOutputStream.MINIMUM_MULTIPART_SIZE) {
		throw new IOException("Provided buffer must be at least " + S3DataOutputStream.MINIMUM_MULTIPART_SIZE
			+ " bytes");
	}

	final byte[] buf = new byte[bufferSize]; // TODO: Use memory manager to allocate larger pages

	return new S3DataOutputStream(this.s3Client, bop.getBucket(), bop.getObject(), buf, this.useRRS);
}
 

import org.apache.flink.core.fs.FSDataOutputStream; //导入依赖的package包/类
@Override
public void close() throws IOException {
	final FSDataOutputStream s = this.stream;
	if (s != null) {
		this.stream = null;
		s.close();
	}
}
 

import org.apache.flink.core.fs.FSDataOutputStream; //导入依赖的package包/类
@Override
public void snapshotState(FSDataOutputStream out, long checkpointId, long timestamp) throws Exception {
	final ObjectOutputStream oos = new ObjectOutputStream(out);

	oos.writeObject(nfa);
	oos.writeInt(priorityQueue.size());

	for (StreamRecord<IN> streamRecord: priorityQueue) {
		streamRecordSerializer.serialize(streamRecord, new DataOutputViewStreamWrapper(oos));
	}
	oos.flush();
}
 

import org.apache.flink.core.fs.FSDataOutputStream; //导入依赖的package包/类
@Test
public void testDirectoryListing() throws Exception {
	final Configuration conf = new Configuration();
	conf.setString("s3.access.key", ACCESS_KEY);
	conf.setString("s3.secret.key", SECRET_KEY);

	FileSystem.initialize(conf);

	final Path directory = new Path("s3://" + BUCKET + '/' + TEST_DATA_DIR + "/testdir/");
	final FileSystem fs = directory.getFileSystem();

	// directory must not yet exist
	assertFalse(fs.exists(directory));

	try {
		// create directory
		assertTrue(fs.mkdirs(directory));

		// seems the presto file system does not assume existence of empty directories in S3
		assertTrue(fs.exists(directory));

		// directory empty
		assertEquals(0, fs.listStatus(directory).length);

		// create some files
		final int numFiles = 3;
		for (int i = 0; i < numFiles; i++) {
			Path file = new Path(directory, "/file-" + i);
			try (FSDataOutputStream out = fs.create(file, WriteMode.NO_OVERWRITE);
					OutputStreamWriter writer = new OutputStreamWriter(out, StandardCharsets.UTF_8)) {
				writer.write("hello-" + i + "\n");
			}
		}

		FileStatus[] files = fs.listStatus(directory);
		assertNotNull(files);
		assertEquals(3, files.length);

		for (FileStatus status : files) {
			assertFalse(status.isDir());
		}

		// now that there are files, the directory must exist
		assertTrue(fs.exists(directory));
	}
	finally {
		// clean up
		fs.delete(directory, true);
	}

	// now directory must be gone
	assertFalse(fs.exists(directory));
}
 

import org.apache.flink.core.fs.FSDataOutputStream; //导入依赖的package包/类
@Test
public void testSimpleFileWriteAndRead() throws Exception {
	final Configuration conf = new Configuration();
	conf.setString("s3.access-key", ACCESS_KEY);
	conf.setString("s3.secret-key", SECRET_KEY);

	final String testLine = "Hello Upload!";

	FileSystem.initialize(conf);

	final Path path = new Path("s3://" + BUCKET + '/' + TEST_DATA_DIR + "/test.txt");
	final FileSystem fs = path.getFileSystem();

	try {
		try (FSDataOutputStream out = fs.create(path, WriteMode.OVERWRITE);
				OutputStreamWriter writer = new OutputStreamWriter(out, StandardCharsets.UTF_8)) {
			writer.write(testLine);
		}

		try (FSDataInputStream in = fs.open(path);
				InputStreamReader ir = new InputStreamReader(in, StandardCharsets.UTF_8);
				BufferedReader reader = new BufferedReader(ir)) {
			String line = reader.readLine();
			assertEquals(testLine, line);
		}
	}
	finally {
		fs.delete(path, false);
	}
}
 

import org.apache.flink.core.fs.FSDataOutputStream; //导入依赖的package包/类
@Test
	public void testDirectoryListing() throws Exception {
		final Configuration conf = new Configuration();
		conf.setString("s3.access-key", ACCESS_KEY);
		conf.setString("s3.secret-key", SECRET_KEY);

		FileSystem.initialize(conf);

		final Path directory = new Path("s3://" + BUCKET + '/' + TEST_DATA_DIR + "/testdir/");
		final FileSystem fs = directory.getFileSystem();

		// directory must not yet exist
		assertFalse(fs.exists(directory));

		try {
			// create directory
			assertTrue(fs.mkdirs(directory));

			// seems the presto file system does not assume existence of empty directories in S3
//			assertTrue(fs.exists(directory));

			// directory empty
			assertEquals(0, fs.listStatus(directory).length);

			// create some files
			final int numFiles = 3;
			for (int i = 0; i < numFiles; i++) {
				Path file = new Path(directory, "/file-" + i);
				try (FSDataOutputStream out = fs.create(file, WriteMode.NO_OVERWRITE);
						OutputStreamWriter writer = new OutputStreamWriter(out, StandardCharsets.UTF_8)) {
					writer.write("hello-" + i + "\n");
				}
			}

			FileStatus[] files = fs.listStatus(directory);
			assertNotNull(files);
			assertEquals(3, files.length);

			for (FileStatus status : files) {
				assertFalse(status.isDir());
			}

			// now that there are files, the directory must exist
			assertTrue(fs.exists(directory));
		}
		finally {
			// clean up
			fs.delete(directory, true);
		}

		// now directory must be gone
		assertFalse(fs.exists(directory));
	}