Java SnapshotDiffReport类代码示例

import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport; //导入依赖的package包/类
static SnapshotDiffReport getSnapshotDiffReport(FSDirectory fsd,
    SnapshotManager snapshotManager, String path,
    String fromSnapshot, String toSnapshot) throws IOException {
  SnapshotDiffReport diffs;
  final FSPermissionChecker pc = fsd.getPermissionChecker();
  fsd.readLock();
  try {
    if (fsd.isPermissionEnabled()) {
      checkSubtreeReadPermission(fsd, pc, path, fromSnapshot);
      checkSubtreeReadPermission(fsd, pc, path, toSnapshot);
    }
    INodesInPath iip = fsd.getINodesInPath(path, true);
    diffs = snapshotManager.diff(iip, path, fromSnapshot, toSnapshot);
  } finally {
    fsd.readUnlock();
  }
  return diffs;
}
 

import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport; //导入依赖的package包/类
/**
 * Get the difference between two snapshots (or between a snapshot and the
 * current status) of a snapshottable directory.
 * 
 * @param path The full path of the snapshottable directory.
 * @param fromSnapshot Name of the snapshot to calculate the diff from. Null
 *          or empty string indicates the current tree.
 * @param toSnapshot Name of the snapshot to calculated the diff to. Null or
 *          empty string indicates the current tree.
 * @return A report about the difference between {@code fromSnapshot} and 
 *         {@code toSnapshot}. Modified/deleted/created/renamed files and 
 *         directories belonging to the snapshottable directories are listed 
 *         and labeled as M/-/+/R respectively. 
 * @throws IOException
 */
SnapshotDiffReport getSnapshotDiffReport(String path,
    String fromSnapshot, String toSnapshot) throws IOException {
  SnapshotDiffReport diffs = null;
  checkOperation(OperationCategory.READ);
  readLock();
  try {
    checkOperation(OperationCategory.READ);
    diffs = FSDirSnapshotOp.getSnapshotDiffReport(dir, snapshotManager,
        path, fromSnapshot, toSnapshot);
  } finally {
    readUnlock();
  }

  logAuditEvent(diffs != null, "computeSnapshotDiff", null, null, null);
  return diffs;
}
 

import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport; //导入依赖的package包/类
/**
 * Compute the difference between two snapshots of a directory, or between a
 * snapshot of the directory and its current tree.
 */
public SnapshotDiffReport diff(final INodesInPath iip,
    final String snapshotRootPath, final String from,
    final String to) throws IOException {
  // Find the source root directory path where the snapshots were taken.
  // All the check for path has been included in the valueOf method.
  final INodeDirectory snapshotRoot = getSnapshottableRoot(iip);

  if ((from == null || from.isEmpty())
      && (to == null || to.isEmpty())) {
    // both fromSnapshot and toSnapshot indicate the current tree
    return new SnapshotDiffReport(snapshotRootPath, from, to,
        Collections.<DiffReportEntry> emptyList());
  }
  final SnapshotDiffInfo diffs = snapshotRoot
      .getDirectorySnapshottableFeature().computeDiff(snapshotRoot, from, to);
  return diffs != null ? diffs.generateReport() : new SnapshotDiffReport(
      snapshotRootPath, from, to, Collections.<DiffReportEntry> emptyList());
}
 

import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport; //导入依赖的package包/类
/**
 * Generate a {@link SnapshotDiffReport} based on detailed diff information.
 * @return A {@link SnapshotDiffReport} describing the difference
 */
public SnapshotDiffReport generateReport() {
  List<DiffReportEntry> diffReportList = new ArrayList<DiffReportEntry>();
  for (Map.Entry<INode,byte[][]> drEntry : diffMap.entrySet()) {
    INode node = drEntry.getKey();
    byte[][] path = drEntry.getValue();
    diffReportList.add(new DiffReportEntry(DiffType.MODIFY, path, null));
    if (node.isDirectory()) {
      List<DiffReportEntry> subList = generateReport(dirDiffMap.get(node),
          path, isFromEarlier(), renameMap);
      diffReportList.addAll(subList);
    }
  }
  return new SnapshotDiffReport(snapshotRoot.getFullPathName(),
      Snapshot.getSnapshotName(from), Snapshot.getSnapshotName(to),
      diffReportList);
}
 

import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport; //导入依赖的package包/类
public static SnapshotDiffReport convert(SnapshotDiffReportProto reportProto) {
  if (reportProto == null) {
    return null;
  }
  String snapshotDir = reportProto.getSnapshotRoot();
  String fromSnapshot = reportProto.getFromSnapshot();
  String toSnapshot = reportProto.getToSnapshot();
  List<SnapshotDiffReportEntryProto> list = reportProto
      .getDiffReportEntriesList();
  List<DiffReportEntry> entries = new ArrayList<DiffReportEntry>();
  for (SnapshotDiffReportEntryProto entryProto : list) {
    DiffReportEntry entry = convert(entryProto);
    if (entry != null)
      entries.add(entry);
  }
  return new SnapshotDiffReport(snapshotDir, fromSnapshot, toSnapshot,
      entries);
}
 

import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport; //导入依赖的package包/类
/**
 * Rename a file under a snapshottable directory, file does not exist
 * in a snapshot.
 */
@Test (timeout=60000)
public void testRenameFileNotInSnapshot() throws Exception {
  hdfs.mkdirs(sub1);
  hdfs.allowSnapshot(sub1);
  hdfs.createSnapshot(sub1, snap1);
  DFSTestUtil.createFile(hdfs, file1, BLOCKSIZE, REPL, SEED);
  hdfs.rename(file1, file2);

  // Query the diff report and make sure it looks as expected.
  SnapshotDiffReport diffReport = hdfs.getSnapshotDiffReport(sub1, snap1, "");
  List<DiffReportEntry> entries = diffReport.getDiffList();
  assertTrue(entries.size() == 2);
  assertTrue(existsInDiffReport(entries, DiffType.MODIFY, "", null));
  assertTrue(existsInDiffReport(entries, DiffType.CREATE, file2.getName(),
      null));
}
 

import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport; //导入依赖的package包/类
/**
 * Rename a file under a snapshottable directory, file exists
 * in a snapshot.
 */
@Test
public void testRenameFileInSnapshot() throws Exception {
  hdfs.mkdirs(sub1);
  hdfs.allowSnapshot(sub1);
  DFSTestUtil.createFile(hdfs, file1, BLOCKSIZE, REPL, SEED);
  hdfs.createSnapshot(sub1, snap1);
  hdfs.rename(file1, file2);

  // Query the diff report and make sure it looks as expected.
  SnapshotDiffReport diffReport = hdfs.getSnapshotDiffReport(sub1, snap1, "");
  System.out.println("DiffList is " + diffReport.toString());
  List<DiffReportEntry> entries = diffReport.getDiffList();
  assertTrue(entries.size() == 2);
  assertTrue(existsInDiffReport(entries, DiffType.MODIFY, "", null));
  assertTrue(existsInDiffReport(entries, DiffType.RENAME, file1.getName(),
      file2.getName()));
}
 

import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport; //导入依赖的package包/类
@Test (timeout=60000)
public void testRenameFileInSubDirOfDirWithSnapshot() throws Exception {
  final Path sub2 = new Path(sub1, "sub2");
  final Path sub2file1 = new Path(sub2, "sub2file1");
  final Path sub2file2 = new Path(sub2, "sub2file2");
  final String sub1snap1 = "sub1snap1";
  
  hdfs.mkdirs(sub1);
  hdfs.mkdirs(sub2);
  DFSTestUtil.createFile(hdfs, sub2file1, BLOCKSIZE, REPL, SEED);
  SnapshotTestHelper.createSnapshot(hdfs, sub1, sub1snap1);

  // Rename the file in the subdirectory.
  hdfs.rename(sub2file1, sub2file2);

  // Query the diff report and make sure it looks as expected.
  SnapshotDiffReport diffReport = hdfs.getSnapshotDiffReport(sub1, sub1snap1,
      "");
  LOG.info("DiffList is \n\"" + diffReport.toString() + "\"");
  List<DiffReportEntry> entries = diffReport.getDiffList();
  assertTrue(existsInDiffReport(entries, DiffType.MODIFY, sub2.getName(),
      null));
  assertTrue(existsInDiffReport(entries, DiffType.RENAME, sub2.getName()
      + "/" + sub2file1.getName(), sub2.getName() + "/" + sub2file2.getName()));
}
 

import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport; //导入依赖的package包/类
@Test (timeout=60000)
public void testRenameDirectoryInSnapshot() throws Exception {
  final Path sub2 = new Path(sub1, "sub2");
  final Path sub3 = new Path(sub1, "sub3");
  final Path sub2file1 = new Path(sub2, "sub2file1");
  final String sub1snap1 = "sub1snap1";
  
  hdfs.mkdirs(sub1);
  hdfs.mkdirs(sub2);
  DFSTestUtil.createFile(hdfs, sub2file1, BLOCKSIZE, REPL, SEED);
  SnapshotTestHelper.createSnapshot(hdfs, sub1, sub1snap1);
  
  // First rename the sub-directory.
  hdfs.rename(sub2, sub3);
  
  // Query the diff report and make sure it looks as expected.
  SnapshotDiffReport diffReport = hdfs.getSnapshotDiffReport(sub1, sub1snap1,
      "");
  LOG.info("DiffList is \n\"" + diffReport.toString() + "\"");
  List<DiffReportEntry> entries = diffReport.getDiffList();
  assertEquals(2, entries.size());
  assertTrue(existsInDiffReport(entries, DiffType.MODIFY, "", null));
  assertTrue(existsInDiffReport(entries, DiffType.RENAME, sub2.getName(),
      sub3.getName()));
}
 

import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport; //导入依赖的package包/类
/**
 * Compute the snapshot diff on the given file system. Return true if the diff
 * is empty, i.e., no changes have happened in the FS.
 */
private static boolean checkNoChange(DistCpOptions inputOptions,
    DistributedFileSystem fs, Path path) {
  try {
    SnapshotDiffReport targetDiff =
        fs.getSnapshotDiffReport(path, inputOptions.getFromSnapshot(), "");
    if (!targetDiff.getDiffList().isEmpty()) {
      DistCp.LOG.warn("The target has been modified since snapshot "
          + inputOptions.getFromSnapshot());
      return false;
    } else {
      return true;
    }
  } catch (IOException e) {
    DistCp.LOG.warn("Failed to compute snapshot diff on " + path, e);
  }
  return false;
}
 

import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport; //导入依赖的package包/类
@Test
public void testSync2() throws Exception {
  initData2(source);
  initData2(target);
  dfs.allowSnapshot(source);
  dfs.allowSnapshot(target);
  dfs.createSnapshot(source, "s1");
  dfs.createSnapshot(target, "s1");

  // make changes under source
  changeData2(source);
  dfs.createSnapshot(source, "s2");

  SnapshotDiffReport report = dfs.getSnapshotDiffReport(source, "s1", "s2");
  System.out.println(report);

  // do the sync
  Assert.assertTrue(DistCpSync.sync(options, conf));
  verifyCopy(dfs.getFileStatus(source), dfs.getFileStatus(target), false);
}
 

import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport; //导入依赖的package包/类
/**
 * Test a case where there are multiple source files with the same name
 */
@Test
public void testSync3() throws Exception {
  initData3(source);
  initData3(target);
  dfs.allowSnapshot(source);
  dfs.allowSnapshot(target);
  dfs.createSnapshot(source, "s1");
  dfs.createSnapshot(target, "s1");

  // make changes under source
  changeData3(source);
  dfs.createSnapshot(source, "s2");

  SnapshotDiffReport report = dfs.getSnapshotDiffReport(source, "s1", "s2");
  System.out.println(report);

  // do the sync
  Assert.assertTrue(DistCpSync.sync(options, conf));
  verifyCopy(dfs.getFileStatus(source), dfs.getFileStatus(target), false);
}
 

import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport; //导入依赖的package包/类
public static SnapshotDiffReport convert(
    SnapshotDiffReportProto reportProto) {
  if (reportProto == null) {
    return null;
  }
  String snapshotDir = reportProto.getSnapshotRoot();
  String fromSnapshot = reportProto.getFromSnapshot();
  String toSnapshot = reportProto.getToSnapshot();
  List<SnapshotDiffReportEntryProto> list = reportProto
      .getDiffReportEntriesList();
  List<DiffReportEntry> entries = new ArrayList<>();
  for (SnapshotDiffReportEntryProto entryProto : list) {
    DiffReportEntry entry = convert(entryProto);
    if (entry != null)
      entries.add(entry);
  }
  return new SnapshotDiffReport(snapshotDir, fromSnapshot, toSnapshot,
      entries);
}
 

import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport; //导入依赖的package包/类
/**
 * Get the difference between two snapshots (or between a snapshot and the
 * current status) of a snapshottable directory.
 * 
 * @param path The full path of the snapshottable directory.
 * @param fromSnapshot Name of the snapshot to calculate the diff from. Null
 *          or empty string indicates the current tree.
 * @param toSnapshot Name of the snapshot to calculated the diff to. Null or
 *          empty string indicates the current tree.
 * @return A report about the difference between {@code fromSnapshot} and 
 *         {@code toSnapshot}. Modified/deleted/created/renamed files and 
 *         directories belonging to the snapshottable directories are listed 
 *         and labeled as M/-/+/R respectively. 
 * @throws IOException
 */
SnapshotDiffReport getSnapshotDiffReport(String path,
    String fromSnapshot, String toSnapshot) throws IOException {
  SnapshotDiffReport diffs = null;
  checkOperation(OperationCategory.READ);
  readLock();
  try {
    checkOperation(OperationCategory.READ);
    diffs = FSDirSnapshotOp.getSnapshotDiffReport(dir, snapshotManager,
        path, fromSnapshot, toSnapshot);
  } finally {
    readUnlock();
  }
  String fromSnapshotRoot = (fromSnapshot == null || fromSnapshot.isEmpty()) ?
      path : Snapshot.getSnapshotPath(path, fromSnapshot);
  String toSnapshotRoot = (toSnapshot == null || toSnapshot.isEmpty()) ?
      path : Snapshot.getSnapshotPath(path, toSnapshot);
  logAuditEvent(diffs != null, "computeSnapshotDiff", fromSnapshotRoot,
      toSnapshotRoot, null);
  return diffs;
}
 

import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport; //导入依赖的package包/类
/**
 * Compute the snapshot diff on the given file system. Return true if the diff
 * is empty, i.e., no changes have happened in the FS.
 */
private boolean checkNoChange(DistributedFileSystem fs, Path path) {
  try {
    SnapshotDiffReport targetDiff =
        fs.getSnapshotDiffReport(path, inputOptions.getFromSnapshot(), "");
    if (!targetDiff.getDiffList().isEmpty()) {
      DistCp.LOG.warn("The target has been modified since snapshot "
          + inputOptions.getFromSnapshot());
      return false;
    } else {
      return true;
    }
  } catch (IOException e) {
    DistCp.LOG.warn("Failed to compute snapshot diff on " + path, e);
  }
  return false;
}