Java FileChecksum.equals方法代码示例

import org.apache.hadoop.fs.FileChecksum; //导入方法依赖的package包/类
/**
 * Check if the two files are equal by looking at the file length,
 * and at the checksum (if user has specified the verifyChecksum flag).
 */
private boolean sameFile(final FileStatus inputStat, final FileStatus outputStat) {
  // Not matching length
  if (inputStat.getLen() != outputStat.getLen()) return false;

  // Mark files as equals, since user asked for no checksum verification
  if (!verifyChecksum) return true;

  // If checksums are not available, files are not the same.
  FileChecksum inChecksum = getFileChecksum(inputFs, inputStat.getPath());
  if (inChecksum == null) return false;

  FileChecksum outChecksum = getFileChecksum(outputFs, outputStat.getPath());
  if (outChecksum == null) return false;

  return inChecksum.equals(outChecksum);
}
 

import org.apache.hadoop.fs.FileChecksum; //导入方法依赖的package包/类
private FileAction checkUpdate(FileSystem sourceFS, FileStatus source,
    Path target, FileStatus targetFileStatus) throws IOException {
  if (targetFileStatus != null && !overWrite) {
    if (canSkip(sourceFS, source, targetFileStatus)) {
      return FileAction.SKIP;
    } else if (append) {
      long targetLen = targetFileStatus.getLen();
      if (targetLen < source.getLen()) {
        FileChecksum sourceChecksum = sourceFS.getFileChecksum(
            source.getPath(), targetLen);
        if (sourceChecksum != null
            && sourceChecksum.equals(targetFS.getFileChecksum(target))) {
          // We require that the checksum is not null. Thus currently only
          // DistributedFileSystem is supported
          return FileAction.APPEND;
        }
      }
    }
  }
  return FileAction.OVERWRITE;
}
 

import org.apache.hadoop.fs.FileChecksum; //导入方法依赖的package包/类
private FileAction checkUpdate(FileSystem sourceFS, FileStatus source,
    Path target) throws IOException {
  final FileStatus targetFileStatus;
  try {
    targetFileStatus = targetFS.getFileStatus(target);
  } catch (FileNotFoundException e) {
    return FileAction.OVERWRITE;
  }
  if (targetFileStatus != null && !overWrite) {
    if (canSkip(sourceFS, source, targetFileStatus)) {
      return FileAction.SKIP;
    } else if (append) {
      long targetLen = targetFileStatus.getLen();
      if (targetLen < source.getLen()) {
        FileChecksum sourceChecksum = sourceFS.getFileChecksum(
            source.getPath(), targetLen);
        if (sourceChecksum != null
            && sourceChecksum.equals(targetFS.getFileChecksum(target))) {
          // We require that the checksum is not null. Thus currently only
          // DistributedFileSystem is supported
          return FileAction.APPEND;
        }
      }
    }
  }
  return FileAction.OVERWRITE;
}
 

import org.apache.hadoop.fs.FileChecksum; //导入方法依赖的package包/类
public void map(Text source, Text target, OutputCollector<NullWritable, NullWritable> oc, Reporter rprtr) throws IOException {
    Path sourceFile = new Path(source.toString());
    Path finalFile = new Path(target.toString());
    Path tmpFile = new Path(tmpRoot, UUID.randomUUID().toString());

    setStatus(rprtr, "Copying " + sourceFile.toString() + " to " + tmpFile.toString());

    if(fsDest.exists(finalFile)) {
        FileChecksum fc1 = fsSource.getFileChecksum(sourceFile);
        FileChecksum fc2 = fsDest.getFileChecksum(finalFile);
        if(fc1 != null && fc2 != null && !fc1.equals(fc2) ||
           fsSource.getContentSummary(sourceFile).getLength()!=fsDest.getContentSummary(finalFile).getLength() ||
           ((fc1==null || fc2==null) && !Utils.firstNBytesSame(fsSource, sourceFile, fsDest, finalFile, 1024*1024))) {
            throw new IOException("Target file already exists and is different! " + finalFile.toString());
        } else {
            return;
        }
    }

    fsDest.mkdirs(tmpFile.getParent());

    copyFile(fsSource, sourceFile, fsDest, tmpFile, rprtr);

    setStatus(rprtr, "Renaming " + tmpFile.toString() + " to " + finalFile.toString());

    fsDest.mkdirs(finalFile.getParent());
    if(!fsDest.rename(tmpFile, finalFile))
        throw new IOException("could not rename " + tmpFile.toString() + " to " + finalFile.toString());

    // this is a bit of a hack; if we don't do this explicit rename, the owner of the file will
    // be hadoop each time.
    //fsDest.setOwner(finalFile, this.owner, fs.getGroup());
}
 

import org.apache.hadoop.fs.FileChecksum; //导入方法依赖的package包/类
/**
 * Utility to compare checksums for the paths specified.
 *
 * If checksums's can't be retrieved, it doesn't fail the test
 * Only time the comparison would fail is when checksums are
 * available and they don't match
 *
 * @param sourceFS FileSystem for the source path.
 * @param source The source path.
 * @param sourceChecksum The checksum of the source file. If it is null we
 * still need to retrieve it through sourceFS.
 * @param targetFS FileSystem for the target path.
 * @param target The target path.
 * @return If either checksum couldn't be retrieved, the function returns
 * false. If checksums are retrieved, the function returns true if they match,
 * and false otherwise.
 * @throws IOException if there's an exception while retrieving checksums.
 */
public static boolean checksumsAreEqual(FileSystem sourceFS, Path source,
    FileChecksum sourceChecksum, FileSystem targetFS, Path target)
    throws IOException {
  FileChecksum targetChecksum = null;
  try {
    sourceChecksum = sourceChecksum != null ? sourceChecksum : sourceFS
        .getFileChecksum(source);
    targetChecksum = targetFS.getFileChecksum(target);
  } catch (IOException e) {
    LOG.error("Unable to retrieve checksum for " + source + " or " + target, e);
  }
  return (sourceChecksum == null || targetChecksum == null ||
          sourceChecksum.equals(targetChecksum));
}
 

import org.apache.hadoop.fs.FileChecksum; //导入方法依赖的package包/类
/**
 * Utility to compare checksums for the paths specified.
 *
 * If checksums's can't be retrieved, it doesn't fail the test
 * Only time the comparison would fail is when checksums are
 * available and they don't match
 *                                  
 * @param sourceFS FileSystem for the source path.
 * @param source The source path.
 * @param targetFS FileSystem for the target path.
 * @param target The target path.
 * @return If either checksum couldn't be retrieved, the function returns
 * false. If checksums are retrieved, the function returns true if they match,
 * and false otherwise.
 * @throws IOException if there's an exception while retrieving checksums.
 */
public static boolean checksumsAreEqual(FileSystem sourceFS, Path source,
                                 FileSystem targetFS, Path target)
                                 throws IOException {
  FileChecksum sourceChecksum = null;
  FileChecksum targetChecksum = null;
  try {
    sourceChecksum = sourceFS.getFileChecksum(source);
    targetChecksum = targetFS.getFileChecksum(target);
  } catch (IOException e) {
    LOG.error("Unable to retrieve checksum for " + source + " or " + target, e);
  }
  return (sourceChecksum == null || targetChecksum == null ||
          sourceChecksum.equals(targetChecksum));
}