Java FileStatus.getLen方法代码示例

import org.apache.flink.core.fs.FileStatus; //导入方法依赖的package包/类
protected FileBaseStatistics getFileStats(FileBaseStatistics cachedStats, Path filePath, FileSystem fs,
		ArrayList<FileStatus> files) throws IOException {
	
	// get the file info and check whether the cached statistics are still valid.
	final FileStatus file = fs.getFileStatus(filePath);
	long totalLength = 0;

	// enumerate all files
	if (file.isDir()) {
		totalLength += addFilesInDir(file.getPath(), files, false);
	} else {
		files.add(file);
		testForUnsplittable(file);
		totalLength += file.getLen();
	}

	// check the modification time stamp
	long latestModTime = 0;
	for (FileStatus f : files) {
		latestModTime = Math.max(f.getModificationTime(), latestModTime);
	}

	// check whether the cached statistics are still valid, if we have any
	if (cachedStats != null && latestModTime <= cachedStats.getLastModificationTime()) {
		return cachedStats;
	}

	// sanity check
	if (totalLength <= 0) {
		totalLength = BaseStatistics.SIZE_UNKNOWN;
	}
	return new FileBaseStatistics(latestModTime, totalLength, BaseStatistics.AVG_RECORD_BYTES_UNKNOWN);
}
 

import org.apache.flink.core.fs.FileStatus; //导入方法依赖的package包/类
/**
 * Enumerate all files in the directory and recursive if enumerateNestedFiles is true.
 * @return the total length of accepted files.
 */
private long addFilesInDir(Path path, List<FileStatus> files, boolean logExcludedFiles)
		throws IOException {
	final FileSystem fs = path.getFileSystem();

	long length = 0;

	for(FileStatus dir: fs.listStatus(path)) {
		if (dir.isDir()) {
			if (acceptFile(dir) && enumerateNestedFiles) {
				length += addFilesInDir(dir.getPath(), files, logExcludedFiles);
			} else {
				if (logExcludedFiles && LOG.isDebugEnabled()) {
					LOG.debug("Directory "+dir.getPath().toString()+" did not pass the file-filter and is excluded.");
				}
			}
		}
		else {
			if(acceptFile(dir)) {
				files.add(dir);
				length += dir.getLen();
				testForUnsplittable(dir);
			} else {
				if (logExcludedFiles && LOG.isDebugEnabled()) {
					LOG.debug("Directory "+dir.getPath().toString()+" did not pass the file-filter and is excluded.");
				}
			}
		}
	}
	return length;
}
 

import org.apache.flink.core.fs.FileStatus; //导入方法依赖的package包/类
/**
 * Fill in the statistics. The last modification time and the total input size are prefilled.
 *
 * @param files
 *        The files that are associated with this block input format.
 * @param stats
 *        The pre-filled statistics.
 */
protected SequentialStatistics createStatistics(List<FileStatus> files, FileBaseStatistics stats)
		throws IOException {
	if (files.isEmpty()) {
		return null;
	}

	BlockInfo blockInfo = new BlockInfo();
	long totalCount = 0;
	for (FileStatus file : files) {
		// invalid file
		if (file.getLen() < blockInfo.getInfoSize()) {
			continue;
		}

		FileSystem fs = file.getPath().getFileSystem();
		try (FSDataInputStream fdis = fs.open(file.getPath(), blockInfo.getInfoSize())) {
			fdis.seek(file.getLen() - blockInfo.getInfoSize());

			blockInfo.read(new DataInputViewStreamWrapper(fdis));
			totalCount += blockInfo.getAccumulatedRecordCount();
		}
	}

	final float avgWidth = totalCount == 0 ? 0 : ((float) stats.getTotalInputSize() / totalCount);
	return new SequentialStatistics(stats.getLastModificationTime(), stats.getTotalInputSize(), avgWidth,
		totalCount);
}
 

import org.apache.flink.core.fs.FileStatus; //导入方法依赖的package包/类
/**
 * Fill in the statistics. The last modification time and the total input size are prefilled.
 * 
 * @param files
 *        The files that are associated with this block input format.
 * @param stats
 *        The pre-filled statistics.
 */
protected SequentialStatistics createStatistics(List<FileStatus> files, FileBaseStatistics stats)
		throws IOException {
	if (files.isEmpty()) {
		return null;
	}

	BlockInfo blockInfo = this.createBlockInfo();

	long totalCount = 0;
	for (FileStatus file : files) {
		// invalid file
		if (file.getLen() < blockInfo.getInfoSize()) {
			continue;
		}

		FSDataInputStream fdis = file.getPath().getFileSystem().open(file.getPath(), blockInfo.getInfoSize());
		fdis.seek(file.getLen() - blockInfo.getInfoSize());

		DataInputStream input = new DataInputStream(fdis);
		blockInfo.read(new InputViewDataInputStreamWrapper(input));
		totalCount += blockInfo.getAccumulatedRecordCount();
	}

	final float avgWidth = totalCount == 0 ? 0 : ((float) stats.getTotalInputSize() / totalCount);
	return new SequentialStatistics(stats.getLastModificationTime(), stats.getTotalInputSize(), avgWidth,
		totalCount);
}
 

import org.apache.flink.core.fs.FileStatus; //导入方法依赖的package包/类
@Override
public BlockLocation[] getFileBlockLocations(final FileStatus file, final long start, final long len)
		throws IOException {

	final BlockLocation[] blockLocations = new BlockLocation[1];
	blockLocations[0] = new LocalBlockLocation(this.hostName, file.getLen());

	return blockLocations;
}
 

import org.apache.flink.core.fs.FileStatus; //导入方法依赖的package包/类
@Override
public BlockLocation[] getFileBlockLocations(final FileStatus file, final long start, final long len)
		throws IOException {

	if ((start + len) > file.getLen()) {
		return null;
	}

	final S3BlockLocation bl = new S3BlockLocation(this.host, file.getLen());

	return new BlockLocation[] { bl };
}
 

import org.apache.flink.core.fs.FileStatus; //导入方法依赖的package包/类
@Override
public BlockLocation[] getFileBlockLocations(FileStatus file, long start, long len) throws IOException {
	return new BlockLocation[] {
			new LocalBlockLocation(hostName, file.getLen())
	};
}