Java FsDatasetSpi.FsVolumeReferences方法代码示例

import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi; //导入方法依赖的package包/类
/**
 *
 * @param blockSize
 * @param perVolumeCapacity limit the capacity of each volume to the given
 *                          value. If negative, then don't limit.
 * @throws IOException
 */
private void startCluster(int blockSize, int numDatanodes, long perVolumeCapacity) throws IOException {
  initConfig(blockSize);

  cluster = new MiniDFSCluster
      .Builder(conf)
      .storagesPerDatanode(STORAGES_PER_DATANODE)
      .numDataNodes(numDatanodes)
      .build();
  fs = cluster.getFileSystem();
  client = fs.getClient();
  cluster.waitActive();

  if (perVolumeCapacity >= 0) {
    try (FsDatasetSpi.FsVolumeReferences volumes =
        cluster.getDataNodes().get(0).getFSDataset().getFsVolumeReferences()) {
      singletonVolumeRef = volumes.get(0).obtainReference();
    }
    singletonVolume = ((FsVolumeImpl) singletonVolumeRef.getVolume());
    singletonVolume.setCapacityForTesting(perVolumeCapacity);
  }
}
 

import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi; //导入方法依赖的package包/类
/**
 * Check that the permissions of the local DN directories are as expected.
 */
@Test
public void testLocalDirs() throws Exception {
  Configuration conf = new Configuration();
  final String permStr = conf.get(
    DFSConfigKeys.DFS_DATANODE_DATA_DIR_PERMISSION_KEY);
  FsPermission expected = new FsPermission(permStr);

  // Check permissions on directories in 'dfs.datanode.data.dir'
  FileSystem localFS = FileSystem.getLocal(conf);
  for (DataNode dn : cluster.getDataNodes()) {
    try (FsDatasetSpi.FsVolumeReferences volumes =
        dn.getFSDataset().getFsVolumeReferences()) {
      for (FsVolumeSpi vol : volumes) {
        String dir = vol.getBasePath();
        Path dataDir = new Path(dir);
        FsPermission actual = localFS.getFileStatus(dataDir).getPermission();
        assertEquals("Permission for dir: " + dataDir + ", is " + actual +
            ", while expected is " + expected, expected, actual);
      }
    }
  }
}
 

import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi; //导入方法依赖的package包/类
private void setVolumeFull(DataNode dn, StorageType type) {
  try (FsDatasetSpi.FsVolumeReferences refs = dn.getFSDataset()
      .getFsVolumeReferences()) {
    for (FsVolumeSpi fvs : refs) {
      FsVolumeImpl volume = (FsVolumeImpl) fvs;
      if (volume.getStorageType() == type) {
        LOG.info("setCapacity to 0 for [" + volume.getStorageType() + "]"
            + volume.getStorageID());
        volume.setCapacityForTesting(0);
      }
    }
  } catch (IOException e) {
    LOG.error("Unexpected exception by closing FsVolumeReference", e);
  }
}
 

import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi; //导入方法依赖的package包/类
/**
 * Make sure at least one non-transient volume has a saved copy of the replica.
 * An infinite loop is used to ensure the async lazy persist tasks are completely
 * done before verification. Caller of ensureLazyPersistBlocksAreSaved expects
 * either a successful pass or timeout failure.
 */
protected final void ensureLazyPersistBlocksAreSaved(
    LocatedBlocks locatedBlocks) throws IOException, InterruptedException {
  final String bpid = cluster.getNamesystem().getBlockPoolId();

  final Set<Long> persistedBlockIds = new HashSet<Long>();

  try (FsDatasetSpi.FsVolumeReferences volumes =
      cluster.getDataNodes().get(0).getFSDataset().getFsVolumeReferences()) {
    while (persistedBlockIds.size() < locatedBlocks.getLocatedBlocks()
        .size()) {
      // Take 1 second sleep before each verification iteration
      Thread.sleep(1000);

      for (LocatedBlock lb : locatedBlocks.getLocatedBlocks()) {
        for (FsVolumeSpi v : volumes) {
          if (v.isTransientStorage()) {
            continue;
          }

          FsVolumeImpl volume = (FsVolumeImpl) v;
          File lazyPersistDir =
              volume.getBlockPoolSlice(bpid).getLazypersistDir();

          long blockId = lb.getBlock().getBlockId();
          File targetDir =
              DatanodeUtil.idToBlockDir(lazyPersistDir, blockId);
          File blockFile = new File(targetDir, lb.getBlock().getBlockName());
          if (blockFile.exists()) {
            // Found a persisted copy for this block and added to the Set
            persistedBlockIds.add(blockId);
          }
        }
      }
    }
  }

  // We should have found a persisted copy for each located block.
  assertThat(persistedBlockIds.size(), is(locatedBlocks.getLocatedBlocks().size()));
}
 

import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi; //导入方法依赖的package包/类
protected final boolean verifyDeletedBlocks(LocatedBlocks locatedBlocks)
    throws IOException, InterruptedException {

  LOG.info("Verifying replica has no saved copy after deletion.");
  triggerBlockReport();

  while(
      cluster.getFsDatasetTestUtils(0).getPendingAsyncDeletions()
      > 0L){
    Thread.sleep(1000);
  }

  final String bpid = cluster.getNamesystem().getBlockPoolId();
  final FsDatasetSpi<?> dataset =
      cluster.getDataNodes().get(0).getFSDataset();

  // Make sure deleted replica does not have a copy on either finalized dir of
  // transient volume or finalized dir of non-transient volume
  try (FsDatasetSpi.FsVolumeReferences volumes =
      dataset.getFsVolumeReferences()) {
    for (FsVolumeSpi vol : volumes) {
      FsVolumeImpl volume = (FsVolumeImpl) vol;
      File targetDir = (volume.isTransientStorage()) ?
          volume.getBlockPoolSlice(bpid).getFinalizedDir() :
          volume.getBlockPoolSlice(bpid).getLazypersistDir();
      if (verifyBlockDeletedFromDir(targetDir, locatedBlocks) == false) {
        return false;
      }
    }
  }
  return true;
}
 

import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi; //导入方法依赖的package包/类
/**
 * Ensure that reserved space is released when the client goes away
 * unexpectedly.
 *
 * The verification is done for each replica in the write pipeline.
 *
 * @throws IOException
 */
@Test(timeout=300000)
public void testSpaceReleasedOnUnexpectedEof()
    throws IOException, InterruptedException, TimeoutException {
  final short replication = 3;
  startCluster(BLOCK_SIZE, replication, -1);

  final String methodName = GenericTestUtils.getMethodName();
  final Path file = new Path("/" + methodName + ".01.dat");

  // Write 1 byte to the file and kill the writer.
  FSDataOutputStream os = fs.create(file, replication);
  os.write(new byte[1]);
  os.hsync();
  DFSTestUtil.abortStream((DFSOutputStream) os.getWrappedStream());

  // Ensure all space reserved for the replica was released on each
  // DataNode.
  for (DataNode dn : cluster.getDataNodes()) {
    try (FsDatasetSpi.FsVolumeReferences volumes =
        dn.getFSDataset().getFsVolumeReferences()) {
      final FsVolumeImpl volume = (FsVolumeImpl) volumes.get(0);
      GenericTestUtils.waitFor(new Supplier<Boolean>() {
        @Override
        public Boolean get() {
          return (volume.getReservedForReplicas() == 0);
        }
      }, 500, Integer.MAX_VALUE); // Wait until the test times out.
    }
  }
}
 

import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi; //导入方法依赖的package包/类
private int getNumVolumes() {
  try (FsDatasetSpi.FsVolumeReferences volumes =
      dataset.getFsVolumeReferences()) {
    return volumes.size();
  } catch (IOException e) {
    return 0;
  }
}
 

import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi; //导入方法依赖的package包/类
@Test
public void testAddVolumes() throws IOException {
  final int numNewVolumes = 3;
  final int numExistingVolumes = getNumVolumes();
  final int totalVolumes = numNewVolumes + numExistingVolumes;
  Set<String> expectedVolumes = new HashSet<String>();
  List<NamespaceInfo> nsInfos = Lists.newArrayList();
  for (String bpid : BLOCK_POOL_IDS) {
    nsInfos.add(new NamespaceInfo(0, CLUSTER_ID, bpid, 1));
  }
  for (int i = 0; i < numNewVolumes; i++) {
    String path = BASE_DIR + "/newData" + i;
    String pathUri = new Path(path).toUri().toString();
    expectedVolumes.add(new File(pathUri).toString());
    StorageLocation loc = StorageLocation.parse(pathUri);
    Storage.StorageDirectory sd = createStorageDirectory(new File(path));
    DataStorage.VolumeBuilder builder =
        new DataStorage.VolumeBuilder(storage, sd);
    when(storage.prepareVolume(eq(datanode), eq(loc.getFile()),
        anyListOf(NamespaceInfo.class)))
        .thenReturn(builder);

    dataset.addVolume(loc, nsInfos);
  }

  assertEquals(totalVolumes, getNumVolumes());
  assertEquals(totalVolumes, dataset.storageMap.size());

  Set<String> actualVolumes = new HashSet<String>();
  try (FsDatasetSpi.FsVolumeReferences volumes =
      dataset.getFsVolumeReferences()) {
    for (int i = 0; i < numNewVolumes; i++) {
      actualVolumes.add(volumes.get(numExistingVolumes + i).getBasePath());
    }
  }
  assertEquals(actualVolumes.size(), expectedVolumes.size());
  assertTrue(actualVolumes.containsAll(expectedVolumes));
}
 

import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi; //导入方法依赖的package包/类
@Before
public void startCluster() throws IOException {
  conf = new HdfsConfiguration();
  cluster = new MiniDFSCluster.Builder(conf).numDataNodes(DN_COUNT).build();
  fs = cluster.getFileSystem();
  singletonNn = cluster.getNameNode();
  singletonDn = cluster.getDataNodes().get(0);
  bpos = singletonDn.getAllBpOs().get(0);
  actor = bpos.getBPServiceActors().get(0);
  try (FsDatasetSpi.FsVolumeReferences volumes =
      singletonDn.getFSDataset().getFsVolumeReferences()) {
    storageUuid = volumes.get(0).getStorageID();
  }
}
 

import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi; //导入方法依赖的package包/类
/** Get the FsVolume on the given basePath */
private FsVolumeImpl getVolume(DataNode dn, File basePath)
    throws IOException {
  try (FsDatasetSpi.FsVolumeReferences volumes =
    dn.getFSDataset().getFsVolumeReferences()) {
    for (FsVolumeSpi vol : volumes) {
      if (vol.getBasePath().equals(basePath.getPath())) {
        return (FsVolumeImpl) vol;
      }
    }
  }
  return null;
}
 

import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi; //导入方法依赖的package包/类
/**
 * Duplicate the given block on all volumes.
 * @param blockId
 * @throws IOException
 */
private void duplicateBlock(long blockId) throws IOException {
  synchronized (fds) {
    ReplicaInfo b = FsDatasetTestUtil.fetchReplicaInfo(fds, bpid, blockId);
    try (FsDatasetSpi.FsVolumeReferences volumes =
        fds.getFsVolumeReferences()) {
      for (FsVolumeSpi v : volumes) {
        if (v.getStorageID().equals(b.getVolume().getStorageID())) {
          continue;
        }

        // Volume without a copy of the block. Make a copy now.
        File sourceBlock = b.getBlockFile();
        File sourceMeta = b.getMetaFile();
        String sourceRoot = b.getVolume().getBasePath();
        String destRoot = v.getBasePath();

        String relativeBlockPath =
            new File(sourceRoot).toURI().relativize(sourceBlock.toURI())
                .getPath();
        String relativeMetaPath =
            new File(sourceRoot).toURI().relativize(sourceMeta.toURI())
                .getPath();

        File destBlock = new File(destRoot, relativeBlockPath);
        File destMeta = new File(destRoot, relativeMetaPath);

        destBlock.getParentFile().mkdirs();
        FileUtils.copyFile(sourceBlock, destBlock);
        FileUtils.copyFile(sourceMeta, destMeta);

        if (destBlock.exists() && destMeta.exists()) {
          LOG.info("Copied " + sourceBlock + " ==> " + destBlock);
          LOG.info("Copied " + sourceMeta + " ==> " + destMeta);
        }
      }
    }
  }
}
 

import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi; //导入方法依赖的package包/类
/** Create a block file in a random volume*/
private long createBlockFile() throws IOException {
  long id = getFreeBlockId();
  try (FsDatasetSpi.FsVolumeReferences volumes = fds.getFsVolumeReferences()) {
    int numVolumes = volumes.size();
    int index = rand.nextInt(numVolumes - 1);
    File finalizedDir = volumes.get(index).getFinalizedDir(bpid);
    File file = new File(finalizedDir, getBlockFile(id));
    if (file.createNewFile()) {
      LOG.info("Created block file " + file.getName());
    }
  }
  return id;
}
 

import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi; //导入方法依赖的package包/类
/** Create a metafile in a random volume*/
private long createMetaFile() throws IOException {
  long id = getFreeBlockId();
  try (FsDatasetSpi.FsVolumeReferences refs = fds.getFsVolumeReferences()) {
    int numVolumes = refs.size();
    int index = rand.nextInt(numVolumes - 1);

    File finalizedDir = refs.get(index).getFinalizedDir(bpid);
    File file = new File(finalizedDir, getMetaFile(id));
    if (file.createNewFile()) {
      LOG.info("Created metafile " + file.getName());
    }
  }
  return id;
}
 

import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi; //导入方法依赖的package包/类
/** Create block file and corresponding metafile in a rondom volume */
private long createBlockMetaFile() throws IOException {
  long id = getFreeBlockId();

  try (FsDatasetSpi.FsVolumeReferences refs = fds.getFsVolumeReferences()) {
    int numVolumes = refs.size();
    int index = rand.nextInt(numVolumes - 1);

    File finalizedDir = refs.get(index).getFinalizedDir(bpid);
    File file = new File(finalizedDir, getBlockFile(id));
    if (file.createNewFile()) {
      LOG.info("Created block file " + file.getName());

      // Create files with same prefix as block file but extension names
      // such that during sorting, these files appear around meta file
      // to test how DirectoryScanner handles extraneous files
      String name1 = file.getAbsolutePath() + ".l";
      String name2 = file.getAbsolutePath() + ".n";
      file = new File(name1);
      if (file.createNewFile()) {
        LOG.info("Created extraneous file " + name1);
      }

      file = new File(name2);
      if (file.createNewFile()) {
        LOG.info("Created extraneous file " + name2);
      }

      file = new File(finalizedDir, getMetaFile(id));
      if (file.createNewFile()) {
        LOG.info("Created metafile " + file.getName());
      }
    }
  }
  return id;
}
 

import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi; //导入方法依赖的package包/类
@Test(timeout=120000)
public void testLocatedFileStatusStorageIdsTypes() throws Exception {
  final Configuration conf = getTestConfiguration();
  final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
      .numDataNodes(3).build();
  try {
    final DistributedFileSystem fs = cluster.getFileSystem();
    final Path testFile = new Path("/testListLocatedStatus");
    final int blockSize = 4096;
    final int numBlocks = 10;
    // Create a test file
    final int repl = 2;
    DFSTestUtil.createFile(fs, testFile, blockSize, numBlocks * blockSize,
        blockSize, (short) repl, 0xADDED);
    DFSTestUtil.waitForReplication(fs, testFile, (short) repl, 30000);
    // Get the listing
    RemoteIterator<LocatedFileStatus> it = fs.listLocatedStatus(testFile);
    assertTrue("Expected file to be present", it.hasNext());
    LocatedFileStatus stat = it.next();
    BlockLocation[] locs = stat.getBlockLocations();
    assertEquals("Unexpected number of locations", numBlocks, locs.length);

    Set<String> dnStorageIds = new HashSet<>();
    for (DataNode d : cluster.getDataNodes()) {
      try (FsDatasetSpi.FsVolumeReferences volumes = d.getFSDataset()
          .getFsVolumeReferences()) {
        for (FsVolumeSpi vol : volumes) {
          dnStorageIds.add(vol.getStorageID());
        }
      }
    }

    for (BlockLocation loc : locs) {
      String[] ids = loc.getStorageIds();
      // Run it through a set to deduplicate, since there should be no dupes
      Set<String> storageIds = new HashSet<>();
      Collections.addAll(storageIds, ids);
      assertEquals("Unexpected num storage ids", repl, storageIds.size());
      // Make sure these are all valid storage IDs
      assertTrue("Unknown storage IDs found!", dnStorageIds.containsAll
          (storageIds));
      // Check storage types are the default, since we didn't set any
      StorageType[] types = loc.getStorageTypes();
      assertEquals("Unexpected num storage types", repl, types.length);
      for (StorageType t: types) {
        assertEquals("Unexpected storage type", StorageType.DEFAULT, t);
      }
    }
  } finally {
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}