Java FileSystem.open方法代码示例

import org.apache.flink.core.fs.FileSystem; //导入方法依赖的package包/类
@Override
public void run() {
	try {
		final FileSystem fs = FileSystem.get(this.split.getPath().toUri());
		this.fdis = fs.open(this.split.getPath());
		
		// check for canceling and close the stream in that case, because no one will obtain it
		if (this.aborted) {
			final FSDataInputStream f = this.fdis;
			this.fdis = null;
			f.close();
		}
	}
	catch (Throwable t) {
		this.error = t;
	}
}
 

import org.apache.flink.core.fs.FileSystem; //导入方法依赖的package包/类
@Override
@SuppressWarnings("unchecked")
public StateTable<K, N, SV> deserialize(
		String stateName,
		HeapKeyedStateBackend<K> stateBackend) throws IOException {

	final FileSystem fs = getFilePath().getFileSystem();
	try (FSDataInputStream inStream = fs.open(getFilePath())) {
		final DataInputViewStreamWrapper inView = new DataInputViewStreamWrapper(inStream);
		AbstractMigrationRestoreStrategy<K, N, SV> restoreStrategy =
				new AbstractMigrationRestoreStrategy<K, N, SV>(keySerializer, namespaceSerializer, stateSerializer) {
					@Override
					protected DataInputView openDataInputView() throws IOException {
						return inView;
					}
				};
		return restoreStrategy.deserialize(stateName, stateBackend);
	}
}
 

import org.apache.flink.core.fs.FileSystem; //导入方法依赖的package包/类
@Override
public void open(TweetFileInputSplit split) throws IOException {
    FileSystem fileSystem = getFileSystem();
    this.reader = new BufferedReader(new InputStreamReader(fileSystem.open(split.getPath())));
    // Pre-read next line to easily check if we've reached the end of an input split
    this.nextLine = reader.readLine();
}
 

import org.apache.flink.core.fs.FileSystem; //导入方法依赖的package包/类
/**
 * Fill in the statistics. The last modification time and the total input size are prefilled.
 *
 * @param files
 *        The files that are associated with this block input format.
 * @param stats
 *        The pre-filled statistics.
 */
protected SequentialStatistics createStatistics(List<FileStatus> files, FileBaseStatistics stats)
		throws IOException {
	if (files.isEmpty()) {
		return null;
	}

	BlockInfo blockInfo = new BlockInfo();
	long totalCount = 0;
	for (FileStatus file : files) {
		// invalid file
		if (file.getLen() < blockInfo.getInfoSize()) {
			continue;
		}

		FileSystem fs = file.getPath().getFileSystem();
		try (FSDataInputStream fdis = fs.open(file.getPath(), blockInfo.getInfoSize())) {
			fdis.seek(file.getLen() - blockInfo.getInfoSize());

			blockInfo.read(new DataInputViewStreamWrapper(fdis));
			totalCount += blockInfo.getAccumulatedRecordCount();
		}
	}

	final float avgWidth = totalCount == 0 ? 0 : ((float) stats.getTotalInputSize() / totalCount);
	return new SequentialStatistics(stats.getLastModificationTime(), stats.getTotalInputSize(), avgWidth,
		totalCount);
}
 

import org.apache.flink.core.fs.FileSystem; //导入方法依赖的package包/类
@Test
public void testSimpleFileWriteAndRead() throws Exception {
	final Configuration conf = new Configuration();
	conf.setString("s3.access.key", ACCESS_KEY);
	conf.setString("s3.secret.key", SECRET_KEY);

	final String testLine = "Hello Upload!";

	FileSystem.initialize(conf);

	final Path path = new Path("s3://" + BUCKET + '/' + TEST_DATA_DIR + "/test.txt");
	final FileSystem fs = path.getFileSystem();

	try {
		try (FSDataOutputStream out = fs.create(path, WriteMode.OVERWRITE);
				OutputStreamWriter writer = new OutputStreamWriter(out, StandardCharsets.UTF_8)) {
			writer.write(testLine);
		}

		try (FSDataInputStream in = fs.open(path);
				InputStreamReader ir = new InputStreamReader(in, StandardCharsets.UTF_8);
				BufferedReader reader = new BufferedReader(ir)) {
			String line = reader.readLine();
			assertEquals(testLine, line);
		}
	}
	finally {
		fs.delete(path, false);
	}
}
 

import org.apache.flink.core.fs.FileSystem; //导入方法依赖的package包/类
/**
 * Uploads the JAR files to the {@link PermanentBlobService} of the {@link BlobServer} at the
 * given address with HA as configured.
 *
 * @param serverAddress
 * 		Server address of the {@link BlobServer}
 * @param clientConfig
 * 		Any additional configuration for the blob client
 * @param jobId
 * 		ID of the job this blob belongs to (or <tt>null</tt> if job-unrelated)
 * @param jars
 * 		List of JAR files to upload
 *
 * @throws IOException
 * 		if the upload fails
 */
public static List<PermanentBlobKey> uploadJarFiles(
		InetSocketAddress serverAddress, Configuration clientConfig, JobID jobId, List<Path> jars)
		throws IOException {

	checkNotNull(jobId);

	if (jars.isEmpty()) {
		return Collections.emptyList();
	} else {
		List<PermanentBlobKey> blobKeys = new ArrayList<>();

		try (BlobClient blobClient = new BlobClient(serverAddress, clientConfig)) {
			for (final Path jar : jars) {
				final FileSystem fs = jar.getFileSystem();
				FSDataInputStream is = null;
				try {
					is = fs.open(jar);
					final PermanentBlobKey key =
						(PermanentBlobKey) blobClient.putInputStream(jobId, is, PERMANENT_BLOB);
					blobKeys.add(key);
				} finally {
					if (is != null) {
						is.close();
					}
				}
			}
		}

		return blobKeys;
	}
}
 

import org.apache.flink.core.fs.FileSystem; //导入方法依赖的package包/类
public static void copy(Path sourcePath, Path targetPath, boolean executable) throws IOException {
	// TODO rewrite this to make it participate in the closable registry and the lifecycle of a task.
	// we unwrap the file system to get raw streams without safety net
	FileSystem sFS = FileSystem.getUnguardedFileSystem(sourcePath.toUri());
	FileSystem tFS = FileSystem.getUnguardedFileSystem(targetPath.toUri());
	if (!tFS.exists(targetPath)) {
		if (sFS.getFileStatus(sourcePath).isDir()) {
			tFS.mkdirs(targetPath);
			FileStatus[] contents = sFS.listStatus(sourcePath);
			for (FileStatus content : contents) {
				String distPath = content.getPath().toString();
				if (content.isDir()) {
					if (distPath.endsWith("/")) {
						distPath = distPath.substring(0, distPath.length() - 1);
					}
				}
				String localPath = targetPath.toString() + distPath.substring(distPath.lastIndexOf("/"));
				copy(content.getPath(), new Path(localPath), executable);
			}
		} else {
			try (FSDataOutputStream lfsOutput = tFS.create(targetPath, FileSystem.WriteMode.NO_OVERWRITE); FSDataInputStream fsInput = sFS.open(sourcePath)) {
				IOUtils.copyBytes(fsInput, lfsOutput);
				//noinspection ResultOfMethodCallIgnored
				new File(targetPath.toString()).setExecutable(executable);
			} catch (IOException ioe) {
				LOG.error("could not copy file to local file cache.", ioe);
			}
		}
	}
}
 

import org.apache.flink.core.fs.FileSystem; //导入方法依赖的package包/类
public InputStream openFile(Path path) throws IOException {
		FileSystem fs = FileSystem.get(path.toUri());
		return fs.open(path);
}
 

import org.apache.flink.core.fs.FileSystem; //导入方法依赖的package包/类
private StreamStateHandle materializeStateData(Path filePath) throws Exception {
	FSDataInputStream inputStream = null;
	CheckpointStreamFactory.CheckpointStateOutputStream outputStream = null;

	try {
		final byte[] buffer = new byte[8 * 1024];

		FileSystem backupFileSystem = backupPath.getFileSystem();
		inputStream = backupFileSystem.open(filePath);
		closeableRegistry.registerCloseable(inputStream);

		outputStream = checkpointStreamFactory
			.createCheckpointStateOutputStream(checkpointId, checkpointTimestamp);
		closeableRegistry.registerCloseable(outputStream);

		while (true) {
			int numBytes = inputStream.read(buffer);

			if (numBytes == -1) {
				break;
			}

			outputStream.write(buffer, 0, numBytes);
		}

		StreamStateHandle result = null;
		if (closeableRegistry.unregisterCloseable(outputStream)) {
			result = outputStream.closeAndGetHandle();
			outputStream = null;
		}
		return result;

	} finally {
		if (inputStream != null && closeableRegistry.unregisterCloseable(inputStream)) {
			inputStream.close();
		}

		if (outputStream != null && closeableRegistry.unregisterCloseable(outputStream)) {
			outputStream.close();
		}
	}
}
 

import org.apache.flink.core.fs.FileSystem; //导入方法依赖的package包/类
@Test
public void testSimpleFileWriteAndRead() throws Exception {
	final Configuration conf = new Configuration();
	conf.setString("s3.access-key", ACCESS_KEY);
	conf.setString("s3.secret-key", SECRET_KEY);

	final String testLine = "Hello Upload!";

	FileSystem.initialize(conf);

	final Path path = new Path("s3://" + BUCKET + '/' + TEST_DATA_DIR + "/test.txt");
	final FileSystem fs = path.getFileSystem();

	try {
		try (FSDataOutputStream out = fs.create(path, WriteMode.OVERWRITE);
				OutputStreamWriter writer = new OutputStreamWriter(out, StandardCharsets.UTF_8)) {
			writer.write(testLine);
		}

		try (FSDataInputStream in = fs.open(path);
				InputStreamReader ir = new InputStreamReader(in, StandardCharsets.UTF_8);
				BufferedReader reader = new BufferedReader(ir)) {
			String line = reader.readLine();
			assertEquals(testLine, line);
		}
	}
	finally {
		fs.delete(path, false);
	}
}
 

import org.apache.flink.core.fs.FileSystem; //导入方法依赖的package包/类
/**
 * Loads the savepoint at the specified path. This methods returns the savepoint, as well as the
 * handle to the metadata.
 *
 * @param savepointFileOrDirectory Path to the parent savepoint directory or the meta data file.
 * @param classLoader The class loader used to resolve serialized classes from legacy savepoint formats.
 * @return The loaded savepoint
 *
 * @throws IOException Failures during load are forwarded
 */
public static Tuple2<Savepoint, StreamStateHandle> loadSavepointWithHandle(
		String savepointFileOrDirectory,
		ClassLoader classLoader) throws IOException {
	
	checkNotNull(savepointFileOrDirectory, "savepointFileOrDirectory");
	checkNotNull(classLoader, "classLoader");

	Path path = new Path(savepointFileOrDirectory);

	LOG.info("Loading savepoint from {}", path);

	FileSystem fs = FileSystem.get(path.toUri());

	FileStatus status = fs.getFileStatus(path);

	// If this is a directory, we need to find the meta data file
	if (status.isDir()) {
		Path candidatePath = new Path(path, SAVEPOINT_METADATA_FILE);
		if (fs.exists(candidatePath)) {
			path = candidatePath;
			LOG.info("Using savepoint file in {}", path);
		} else {
			throw new IOException("Cannot find meta data file in directory " + path
					+ ". Please try to load the savepoint directly from the meta data file "
					+ "instead of the directory.");
		}
	}

	// load the savepoint
	final Savepoint savepoint;
	try (DataInputStream dis = new DataInputViewStreamWrapper(fs.open(path))) {
		int magicNumber = dis.readInt();

		if (magicNumber == MAGIC_NUMBER) {
			int version = dis.readInt();

			SavepointSerializer<?> serializer = SavepointSerializers.getSerializer(version);
			savepoint = serializer.deserialize(dis, classLoader);
		} else {
			throw new RuntimeException("Unexpected magic number. This can have multiple reasons: " +
					"(1) You are trying to load a Flink 1.0 savepoint, which is not supported by this " +
					"version of Flink. (2) The file you were pointing to is not a savepoint at all. " +
					"(3) The savepoint file has been corrupted.");
		}
	}

	// construct the stream handle to the metadata file
	// we get the size best-effort
	long size = 0;
	try {
		size = fs.getFileStatus(path).getLen();
	}
	catch (Exception ignored) {
		// we don't know the size, but we don't want to fail the savepoint loading for that
	}
	StreamStateHandle metadataHandle = new FileStateHandle(path, size);

	return new Tuple2<>(savepoint, metadataHandle);
}