Java HdfsConstants.DOT_SNAPSHOT_DIR属性代码示例

private Path createSnapshot() throws IOException {
  LOG.debug("Source table {}.{} has its data located at {}", sourceTable.getDbName(), sourceTable.getTableName(),
      sourceDataPath);

  FileSystem fileSystem = fileSystemFactory.get(sourceDataPath, sourceHiveConf);
  Path snapshotMetaDataPath = new Path(sourceDataPath, HdfsConstants.DOT_SNAPSHOT_DIR);
  Path resolvedLocation = sourceDataPath;
  if (fileSystem.exists(snapshotMetaDataPath)) {
    if (snapshotsDisabled) {
      LOG.info("Path {} can be snapshot, but feature has been disabled.", sourceDataPath);
    } else {
      LOG.debug("Creating source data snapshot: {}, {}", sourceDataPath, eventId);
      // fileSystem.createSnapshot does not return a fully qualified URI.
      resolvedLocation = fileSystem.makeQualified(fileSystem.createSnapshot(sourceDataPath, eventId));
      snapshotPath = resolvedLocation;
    }
  } else {
    LOG.debug("Snapshots not enabled on source location: {}", sourceDataPath);
  }
  return resolvedLocation;
}
 

/**
 * Test the sync returns false in the following scenarios:
 * 1. the source/target dir are not snapshottable dir
 * 2. the source/target does not have the given snapshots
 * 3. changes have been made in target
 */
@Test
public void testFallback() throws Exception {
  // the source/target dir are not snapshottable dir
  Assert.assertFalse(DistCpSync.sync(options, conf));
  // make sure the source path has been updated to the snapshot path
  final Path spath = new Path(source,
      HdfsConstants.DOT_SNAPSHOT_DIR + Path.SEPARATOR + "s2");
  Assert.assertEquals(spath, options.getSourcePaths().get(0));

  // reset source path in options
  options.setSourcePaths(Arrays.asList(source));
  // the source/target does not have the given snapshots
  dfs.allowSnapshot(source);
  dfs.allowSnapshot(target);
  Assert.assertFalse(DistCpSync.sync(options, conf));
  Assert.assertEquals(spath, options.getSourcePaths().get(0));

  // reset source path in options
  options.setSourcePaths(Arrays.asList(source));
  dfs.createSnapshot(source, "s1");
  dfs.createSnapshot(source, "s2");
  dfs.createSnapshot(target, "s1");
  Assert.assertTrue(DistCpSync.sync(options, conf));

  // reset source paths in options
  options.setSourcePaths(Arrays.asList(source));
  // changes have been made in target
  final Path subTarget = new Path(target, "sub");
  dfs.mkdirs(subTarget);
  Assert.assertFalse(DistCpSync.sync(options, conf));
  // make sure the source path has been updated to the snapshot path
  Assert.assertEquals(spath, options.getSourcePaths().get(0));

  // reset source paths in options
  options.setSourcePaths(Arrays.asList(source));
  dfs.delete(subTarget, true);
  Assert.assertTrue(DistCpSync.sync(options, conf));
}
 

/** Verify if the inode name is legal. */
void verifyINodeName(byte[] childName) throws HadoopIllegalArgumentException {
  if (Arrays.equals(HdfsConstants.DOT_SNAPSHOT_DIR_BYTES, childName)) {
    String s = "\"" + HdfsConstants.DOT_SNAPSHOT_DIR + "\" is a reserved name.";
    if (!namesystem.isImageLoaded()) {
      s += "  Please rename it before upgrade.";
    }
    throw new HadoopIllegalArgumentException(s);
  }
}
 

private static Path getSourceSnapshotPath(Path sourceDir, String snapshotName) {
  if (Path.CUR_DIR.equals(snapshotName)) {
    return sourceDir;
  } else {
    return new Path(sourceDir,
        HdfsConstants.DOT_SNAPSHOT_DIR + Path.SEPARATOR + snapshotName);
  }
}
 

/**
 * Test rename to an invalid name (xxx/.snapshot)
 */
@Test
public void testRenameUndo_7() throws Exception {
  final Path root = new Path("/");
  final Path foo = new Path(root, "foo");
  final Path bar = new Path(foo, "bar");
  DFSTestUtil.createFile(hdfs, bar, BLOCKSIZE, REPL, SEED);
  
  // create a snapshot on root
  SnapshotTestHelper.createSnapshot(hdfs, root, snap1);
  
  // rename bar to /foo/.snapshot which is invalid
  final Path invalid = new Path(foo, HdfsConstants.DOT_SNAPSHOT_DIR);
  try {
    hdfs.rename(bar, invalid);
    fail("expect exception since invalid name is used for rename");
  } catch (Exception e) {
    GenericTestUtils.assertExceptionContains("\"" +
        HdfsConstants.DOT_SNAPSHOT_DIR + "\" is a reserved name", e);
  }
  
  // check
  INodeDirectory rootNode = fsdir.getINode4Write(root.toString())
      .asDirectory();
  INodeDirectory fooNode = fsdir.getINode4Write(foo.toString()).asDirectory();
  ReadOnlyList<INode> children = fooNode
      .getChildrenList(Snapshot.CURRENT_STATE_ID);
  assertEquals(1, children.size());
  List<DirectoryDiff> diffList = fooNode.getDiffs().asList();
  assertEquals(1, diffList.size());
  DirectoryDiff diff = diffList.get(0);
  // this diff is generated while renaming
  Snapshot s1 = rootNode.getSnapshot(DFSUtil.string2Bytes(snap1));
  assertEquals(s1.getId(), diff.getSnapshotId());
  // after undo, the diff should be empty
  assertTrue(diff.getChildrenDiff().getList(ListType.DELETED).isEmpty());
  assertTrue(diff.getChildrenDiff().getList(ListType.CREATED).isEmpty());
  
  // bar was converted to filewithsnapshot while renaming
  INodeFile barNode = fsdir.getINode4Write(bar.toString()).asFile();
  assertSame(barNode, children.get(0));
  assertSame(fooNode, barNode.getParent());
  List<FileDiff> barDiffList = barNode.getDiffs().asList();
  assertEquals(1, barDiffList.size());
  FileDiff barDiff = barDiffList.get(0);
  assertEquals(s1.getId(), barDiff.getSnapshotId());
  
  // restart cluster multiple times to make sure the fsimage and edits log are
  // correct. Note that when loading fsimage, foo and bar will be converted 
  // back to normal INodeDirectory and INodeFile since they do not store any 
  // snapshot data
  hdfs.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
  hdfs.saveNamespace();
  hdfs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE);
  cluster.shutdown();
  cluster = new MiniDFSCluster.Builder(conf).format(false)
      .numDataNodes(REPL).build();
  cluster.waitActive();
  restartClusterAndCheckImage(true);
}
 

public static Path getSnapshotRoot(Path snapshottedDir, String snapshotName) {
  return new Path(snapshottedDir, HdfsConstants.DOT_SNAPSHOT_DIR + "/"
      + snapshotName);
}
 

/**
 * Test the basic functionality.
 */
@Test
public void testSync() throws Exception {
  initData(source);
  initData(target);
  dfs.allowSnapshot(source);
  dfs.allowSnapshot(target);
  dfs.createSnapshot(source, "s1");
  dfs.createSnapshot(target, "s1");

  // make changes under source
  changeData(source);
  dfs.createSnapshot(source, "s2");

  // before sync, make some further changes on source. this should not affect
  // the later distcp since we're copying (s2-s1) to target
  final Path toDelete = new Path(source, "foo/d1/foo/f1");
  dfs.delete(toDelete, true);
  final Path newdir = new Path(source, "foo/d1/foo/newdir");
  dfs.mkdirs(newdir);

  // do the sync
  Assert.assertTrue(DistCpSync.sync(options, conf));

  // make sure the source path has been updated to the snapshot path
  final Path spath = new Path(source,
      HdfsConstants.DOT_SNAPSHOT_DIR + Path.SEPARATOR + "s2");
  Assert.assertEquals(spath, options.getSourcePaths().get(0));

  // build copy listing
  final Path listingPath = new Path("/tmp/META/fileList.seq");
  CopyListing listing = new GlobbedCopyListing(conf, new Credentials());
  listing.buildListing(listingPath, options);

  Map<Text, CopyListingFileStatus> copyListing = getListing(listingPath);
  CopyMapper copyMapper = new CopyMapper();
  StubContext stubContext = new StubContext(conf, null, 0);
  Mapper<Text, CopyListingFileStatus, Text, Text>.Context context =
      stubContext.getContext();
  // Enable append
  context.getConfiguration().setBoolean(
      DistCpOptionSwitch.APPEND.getConfigLabel(), true);
  copyMapper.setup(context);
  for (Map.Entry<Text, CopyListingFileStatus> entry : copyListing.entrySet()) {
    copyMapper.map(entry.getKey(), entry.getValue(), context);
  }

  // verify that we only copied new appended data of f2 and the new file f1
  Assert.assertEquals(BLOCK_SIZE * 3, stubContext.getReporter()
      .getCounter(CopyMapper.Counter.BYTESCOPIED).getValue());

  // verify the source and target now has the same structure
  verifyCopy(dfs.getFileStatus(spath), dfs.getFileStatus(target), false);
}