Java BlockListAsLongs.getBlockListAsLongs方法代码示例

import org.apache.hadoop.hdfs.protocol.BlockListAsLongs; //导入方法依赖的package包/类
@Override
public DatanodeCommand blockReport(DatanodeRegistration registration,
    String poolId, StorageBlockReport[] reports, BlockReportContext context)
      throws IOException {
  BlockReportRequestProto.Builder builder = BlockReportRequestProto
      .newBuilder().setRegistration(PBHelper.convert(registration))
      .setBlockPoolId(poolId);
  
  boolean useBlocksBuffer = registration.getNamespaceInfo()
      .isCapabilitySupported(Capability.STORAGE_BLOCK_REPORT_BUFFERS);

  for (StorageBlockReport r : reports) {
    StorageBlockReportProto.Builder reportBuilder = StorageBlockReportProto
        .newBuilder().setStorage(PBHelper.convert(r.getStorage()));
    BlockListAsLongs blocks = r.getBlocks();
    if (useBlocksBuffer) {
      reportBuilder.setNumberOfBlocks(blocks.getNumberOfBlocks());
      reportBuilder.addAllBlocksBuffers(blocks.getBlocksBuffers());
    } else {
      for (long value : blocks.getBlockListAsLongs()) {
        reportBuilder.addBlocks(value);
      }
    }
    builder.addReports(reportBuilder.build());
  }
  builder.setContext(PBHelper.convert(context));
  BlockReportResponseProto resp;
  try {
    resp = rpcProxy.blockReport(NULL_CONTROLLER, builder.build());
  } catch (ServiceException se) {
    throw ProtobufHelper.getRemoteException(se);
  }
  return resp.hasCmd() ? PBHelper.convert(resp.getCmd()) : null;
}
 

import org.apache.hadoop.hdfs.protocol.BlockListAsLongs; //导入方法依赖的package包/类
@Override
public DatanodeCommand blockReport(DatanodeRegistration registration,
    String poolId, StorageBlockReport[] reports, BlockReportContext context)
      throws IOException {
  BlockReportRequestProto.Builder builder = BlockReportRequestProto
      .newBuilder().setRegistration(PBHelper.convert(registration))
      .setBlockPoolId(poolId);
  
  boolean useBlocksBuffer = registration.getNamespaceInfo()
      .isCapabilitySupported(Capability.STORAGE_BLOCK_REPORT_BUFFERS);

  for (StorageBlockReport r : reports) {
    StorageBlockReportProto.Builder reportBuilder = StorageBlockReportProto
        .newBuilder().setStorage(PBHelperClient.convert(r.getStorage()));
    BlockListAsLongs blocks = r.getBlocks();
    if (useBlocksBuffer) {
      reportBuilder.setNumberOfBlocks(blocks.getNumberOfBlocks());
      reportBuilder.addAllBlocksBuffers(blocks.getBlocksBuffers());
    } else {
      for (long value : blocks.getBlockListAsLongs()) {
        reportBuilder.addBlocks(value);
      }
    }
    builder.addReports(reportBuilder.build());
  }
  builder.setContext(PBHelper.convert(context));
  BlockReportResponseProto resp;
  try {
    resp = rpcProxy.blockReport(NULL_CONTROLLER, builder.build());
  } catch (ServiceException se) {
    throw ProtobufHelper.getRemoteException(se);
  }
  return resp.hasCmd() ? PBHelper.convert(resp.getCmd()) : null;
}
 

import org.apache.hadoop.hdfs.protocol.BlockListAsLongs; //导入方法依赖的package包/类
private static StorageBlockReport[] getBlockReports(
    DataNode dn, String bpid, boolean corruptOneBlockGs,
    boolean corruptOneBlockLen) {
  Map<DatanodeStorage, BlockListAsLongs> perVolumeBlockLists =
      dn.getFSDataset().getBlockReports(bpid);

  // Send block report
  StorageBlockReport[] reports =
      new StorageBlockReport[perVolumeBlockLists.size()];
  boolean corruptedGs = false;
  boolean corruptedLen = false;

  int reportIndex = 0;
  for(Map.Entry<DatanodeStorage, BlockListAsLongs> kvPair : perVolumeBlockLists.entrySet()) {
    DatanodeStorage dnStorage = kvPair.getKey();
    BlockListAsLongs blockList = kvPair.getValue();

    // Walk the list of blocks until we find one each to corrupt the
    // generation stamp and length, if so requested.
    for (int i = 0; i < blockList.getNumberOfBlocks(); ++i) {
      if (corruptOneBlockGs && !corruptedGs) {
        blockList.corruptBlockGSForTesting(i, rand);
        LOG.info("Corrupted the GS for block ID " + i);
        corruptedGs = true;
      } else if (corruptOneBlockLen && !corruptedLen) {
        blockList.corruptBlockLengthForTesting(i, rand);
        LOG.info("Corrupted the length for block ID " + i);
        corruptedLen = true;
      } else {
        break;
      }
    }

    reports[reportIndex++] =
        new StorageBlockReport(dnStorage, blockList.getBlockListAsLongs());
  }

  return reports;
}
 

import org.apache.hadoop.hdfs.protocol.BlockListAsLongs; //导入方法依赖的package包/类
@Test
public void testVolumeFailure() throws Exception {
  System.out.println("Data dir: is " +  dataDir.getPath());
 
  
  // Data dir structure is dataDir/data[1-4]/[current,tmp...]
  // data1,2 is for datanode 1, data2,3 - datanode2 
  String filename = "/test.txt";
  Path filePath = new Path(filename);
  
  // we use only small number of blocks to avoid creating subdirs in the data dir..
  int filesize = block_size*blocks_num;
  DFSTestUtil.createFile(fs, filePath, filesize, repl, 1L);
  DFSTestUtil.waitReplication(fs, filePath, repl);
  System.out.println("file " + filename + "(size " +
      filesize + ") is created and replicated");
 
  // fail the volume
  // delete/make non-writable one of the directories (failed volume)
  data_fail = new File(dataDir, "data3");
  failedDir = MiniDFSCluster.getFinalizedDir(dataDir, 
      cluster.getNamesystem().getBlockPoolId());
  if (failedDir.exists() &&
      //!FileUtil.fullyDelete(failedDir)
      !deteteBlocks(failedDir)
      ) {
    throw new IOException("Could not delete hdfs directory '" + failedDir + "'");
  }
  data_fail.setReadOnly();
  failedDir.setReadOnly();
  System.out.println("Deleteing " + failedDir.getPath() + "; exist=" + failedDir.exists());
  
  // access all the blocks on the "failed" DataNode, 
  // we need to make sure that the "failed" volume is being accessed - 
  // and that will cause failure, blocks removal, "emergency" block report
  triggerFailure(filename, filesize);
  
  // make sure a block report is sent 
  DataNode dn = cluster.getDataNodes().get(1); //corresponds to dir data3
  String bpid = cluster.getNamesystem().getBlockPoolId();
  DatanodeRegistration dnR = dn.getDNRegistrationForBP(bpid);
  
  Map<DatanodeStorage, BlockListAsLongs> perVolumeBlockLists =
      dn.getFSDataset().getBlockReports(bpid);

  // Send block report
  StorageBlockReport[] reports =
      new StorageBlockReport[perVolumeBlockLists.size()];

  int reportIndex = 0;
  for(Map.Entry<DatanodeStorage, BlockListAsLongs> kvPair : perVolumeBlockLists.entrySet()) {
      DatanodeStorage dnStorage = kvPair.getKey();
      BlockListAsLongs blockList = kvPair.getValue();
      reports[reportIndex++] =
          new StorageBlockReport(dnStorage, blockList.getBlockListAsLongs());
  }
  
  cluster.getNameNodeRpc().blockReport(dnR, bpid, reports, null);

  // verify number of blocks and files...
  verify(filename, filesize);
  
  // create another file (with one volume failed).
  System.out.println("creating file test1.txt");
  Path fileName1 = new Path("/test1.txt");
  DFSTestUtil.createFile(fs, fileName1, filesize, repl, 1L);
  
  // should be able to replicate to both nodes (2 DN, repl=2)
  DFSTestUtil.waitReplication(fs, fileName1, repl);
  System.out.println("file " + fileName1.getName() + 
      " is created and replicated");
}
 

import org.apache.hadoop.hdfs.protocol.BlockListAsLongs; //导入方法依赖的package包/类
/**
 * Report the list blocks to the Namenode
 * @throws IOException
 */
DatanodeCommand blockReport() throws IOException {
  // send block report if timer has expired.
  DatanodeCommand cmd = null;
  long startTime = now();
  if (startTime - lastBlockReport > dnConf.blockReportInterval) {

    // Flush any block information that precedes the block report. Otherwise
    // we have a chance that we will miss the delHint information
    // or we will report an RBW replica after the BlockReport already reports
    // a FINALIZED one.
    reportReceivedDeletedBlocks();

    // Create block report
    long brCreateStartTime = now();
    BlockListAsLongs bReport = dn.getFSDataset().getBlockReport(
        bpos.getBlockPoolId());

    // Send block report
    long brSendStartTime = now();
    StorageBlockReport[] report = { new StorageBlockReport(
        new DatanodeStorage(bpRegistration.getStorageID()),
        bReport.getBlockListAsLongs()) };
    cmd = bpNamenode.blockReport(bpRegistration, bpos.getBlockPoolId(), report);

    // Log the block report processing stats from Datanode perspective
    long brSendCost = now() - brSendStartTime;
    long brCreateCost = brSendStartTime - brCreateStartTime;
    dn.getMetrics().addBlockReport(brSendCost);
    LOG.info("BlockReport of " + bReport.getNumberOfBlocks()
        + " blocks took " + brCreateCost + " msec to generate and "
        + brSendCost + " msecs for RPC and NN processing");

    // If we have sent the first block report, then wait a random
    // time before we start the periodic block reports.
    if (resetBlockReportTime) {
      lastBlockReport = startTime - DFSUtil.getRandom().nextInt((int)(dnConf.blockReportInterval));
      resetBlockReportTime = false;
    } else {
      /* say the last block report was at 8:20:14. The current report
       * should have started around 9:20:14 (default 1 hour interval).
       * If current time is :
       *   1) normal like 9:20:18, next report should be at 10:20:14
       *   2) unexpected like 11:35:43, next report should be at 12:20:14
       */
      lastBlockReport += (now() - lastBlockReport) /
      dnConf.blockReportInterval * dnConf.blockReportInterval;
    }
    LOG.info("sent block report, processed command:" + cmd);
  }
  return cmd;
}
 

import org.apache.hadoop.hdfs.protocol.BlockListAsLongs; //导入方法依赖的package包/类
@Test
public void testVolumeFailure() throws Exception {
  System.out.println("Data dir: is " +  dataDir.getPath());
 
  
  // Data dir structure is dataDir/data[1-4]/[current,tmp...]
  // data1,2 is for datanode 1, data2,3 - datanode2 
  String filename = "/test.txt";
  Path filePath = new Path(filename);
  
  // we use only small number of blocks to avoid creating subdirs in the data dir..
  int filesize = block_size*blocks_num;
  DFSTestUtil.createFile(fs, filePath, filesize, repl, 1L);
  DFSTestUtil.waitReplication(fs, filePath, repl);
  System.out.println("file " + filename + "(size " +
      filesize + ") is created and replicated");
 
  // fail the volume
  // delete/make non-writable one of the directories (failed volume)
  data_fail = new File(dataDir, "data3");
  failedDir = MiniDFSCluster.getFinalizedDir(dataDir, 
      cluster.getNamesystem().getBlockPoolId());
  if (failedDir.exists() &&
      //!FileUtil.fullyDelete(failedDir)
      !deteteBlocks(failedDir)
      ) {
    throw new IOException("Could not delete hdfs directory '" + failedDir + "'");
  }    
  data_fail.setReadOnly();
  failedDir.setReadOnly();
  System.out.println("Deleteing " + failedDir.getPath() + "; exist=" + failedDir.exists());
  
  // access all the blocks on the "failed" DataNode, 
  // we need to make sure that the "failed" volume is being accessed - 
  // and that will cause failure, blocks removal, "emergency" block report
  triggerFailure(filename, filesize);
  
  // make sure a block report is sent 
  DataNode dn = cluster.getDataNodes().get(1); //corresponds to dir data3
  String bpid = cluster.getNamesystem().getBlockPoolId();
  DatanodeRegistration dnR = dn.getDNRegistrationForBP(bpid);
  
  Map<DatanodeStorage, BlockListAsLongs> perVolumeBlockLists =
      dn.getFSDataset().getBlockReports(bpid);

  // Send block report
  StorageBlockReport[] reports =
      new StorageBlockReport[perVolumeBlockLists.size()];

  int reportIndex = 0;
  for(Map.Entry<DatanodeStorage, BlockListAsLongs> kvPair : perVolumeBlockLists.entrySet()) {
      DatanodeStorage dnStorage = kvPair.getKey();
      BlockListAsLongs blockList = kvPair.getValue();
      reports[reportIndex++] =
          new StorageBlockReport(dnStorage, blockList.getBlockListAsLongs());
  }
  
  cluster.getNameNodeRpc().blockReport(dnR, bpid, reports);

  // verify number of blocks and files...
  verify(filename, filesize);
  
  // create another file (with one volume failed).
  System.out.println("creating file test1.txt");
  Path fileName1 = new Path("/test1.txt");
  DFSTestUtil.createFile(fs, fileName1, filesize, repl, 1L);
  
  // should be able to replicate to both nodes (2 DN, repl=2)
  DFSTestUtil.waitReplication(fs, fileName1, repl);
  System.out.println("file " + fileName1.getName() + 
      " is created and replicated");
}
 

import org.apache.hadoop.hdfs.protocol.BlockListAsLongs; //导入方法依赖的package包/类
@Test
public void testVolumeFailure() throws Exception {
  FileSystem fs = cluster.getFileSystem();
  dataDir = new File(cluster.getDataDirectory());
  System.out.println("Data dir: is " +  dataDir.getPath());
 
  
  // Data dir structure is dataDir/data[1-4]/[current,tmp...]
  // data1,2 is for datanode 1, data2,3 - datanode2 
  String filename = "/test.txt";
  Path filePath = new Path(filename);
  
  // we use only small number of blocks to avoid creating subdirs in the data dir..
  int filesize = block_size*blocks_num;
  DFSTestUtil.createFile(fs, filePath, filesize, repl, 1L);
  DFSTestUtil.waitReplication(fs, filePath, repl);
  System.out.println("file " + filename + "(size " +
      filesize + ") is created and replicated");
 
  // fail the volume
  // delete/make non-writable one of the directories (failed volume)
  data_fail = new File(dataDir, "data3");
  failedDir = MiniDFSCluster.getFinalizedDir(dataDir, 
      cluster.getNamesystem().getBlockPoolId());
  if (failedDir.exists() &&
      //!FileUtil.fullyDelete(failedDir)
      !deteteBlocks(failedDir)
      ) {
    throw new IOException("Could not delete hdfs directory '" + failedDir + "'");
  }    
  data_fail.setReadOnly();
  failedDir.setReadOnly();
  System.out.println("Deleteing " + failedDir.getPath() + "; exist=" + failedDir.exists());
  
  // access all the blocks on the "failed" DataNode, 
  // we need to make sure that the "failed" volume is being accessed - 
  // and that will cause failure, blocks removal, "emergency" block report
  triggerFailure(filename, filesize);
  
  // make sure a block report is sent 
  DataNode dn = cluster.getDataNodes().get(1); //corresponds to dir data3
  String bpid = cluster.getNamesystem().getBlockPoolId();
  DatanodeRegistration dnR = dn.getDNRegistrationForBP(bpid);
  
  Map<DatanodeStorage, BlockListAsLongs> perVolumeBlockLists =
      dn.getFSDataset().getBlockReports(bpid);

  // Send block report
  StorageBlockReport[] reports =
      new StorageBlockReport[perVolumeBlockLists.size()];

  int reportIndex = 0;
  for(Map.Entry<DatanodeStorage, BlockListAsLongs> kvPair : perVolumeBlockLists.entrySet()) {
      DatanodeStorage dnStorage = kvPair.getKey();
      BlockListAsLongs blockList = kvPair.getValue();
      reports[reportIndex++] =
          new StorageBlockReport(dnStorage, blockList.getBlockListAsLongs());
  }
  
  cluster.getNameNodeRpc().blockReport(dnR, bpid, reports);

  // verify number of blocks and files...
  verify(filename, filesize);
  
  // create another file (with one volume failed).
  System.out.println("creating file test1.txt");
  Path fileName1 = new Path("/test1.txt");
  DFSTestUtil.createFile(fs, fileName1, filesize, repl, 1L);
  
  // should be able to replicate to both nodes (2 DN, repl=2)
  DFSTestUtil.waitReplication(fs, fileName1, repl);
  System.out.println("file " + fileName1.getName() + 
      " is created and replicated");
}