Java LocatedBlocks.getLastLocatedBlock方法代码示例

import org.apache.hadoop.hdfs.protocol.LocatedBlocks; //导入方法依赖的package包/类
public static LocatedBlocksProto convert(LocatedBlocks lb) {
  if (lb == null) {
    return null;
  }
  LocatedBlocksProto.Builder builder = 
      LocatedBlocksProto.newBuilder();
  if (lb.getLastLocatedBlock() != null) {
    builder.setLastBlock(PBHelper.convert(lb.getLastLocatedBlock()));
  }
  if (lb.getFileEncryptionInfo() != null) {
    builder.setFileEncryptionInfo(convert(lb.getFileEncryptionInfo()));
  }
  return builder.setFileLength(lb.getFileLength())
      .setUnderConstruction(lb.isUnderConstruction())
      .addAllBlocks(PBHelper.convertLocatedBlock2(lb.getLocatedBlocks()))
      .setIsLastBlockComplete(lb.isLastBlockComplete()).build();
}
 

import org.apache.hadoop.hdfs.protocol.LocatedBlocks; //导入方法依赖的package包/类
public static void checkBlockRecovery(Path p, DistributedFileSystem dfs,
    int attempts, long sleepMs) throws IOException {
  boolean success = false;
  for(int i = 0; i < attempts; i++) {
    LocatedBlocks blocks = getLocatedBlocks(p, dfs);
    boolean noLastBlock = blocks.getLastLocatedBlock() == null;
    if(!blocks.isUnderConstruction() &&
        (noLastBlock || blocks.isLastBlockComplete())) {
      success = true;
      break;
    }
    try { Thread.sleep(sleepMs); } catch (InterruptedException ignored) {}
  }
  assertThat("inode should complete in ~" + sleepMs * attempts + " ms.",
      success, is(true));
}
 

import org.apache.hadoop.hdfs.protocol.LocatedBlocks; //导入方法依赖的package包/类
public static LocatedBlocksProto convert(LocatedBlocks lb) {
  if (lb == null) {
    return null;
  }
  LocatedBlocksProto.Builder builder =
      LocatedBlocksProto.newBuilder();
  if (lb.getLastLocatedBlock() != null) {
    builder.setLastBlock(
        convertLocatedBlock(lb.getLastLocatedBlock()));
  }
  if (lb.getFileEncryptionInfo() != null) {
    builder.setFileEncryptionInfo(convert(lb.getFileEncryptionInfo()));
  }
  if (lb.getErasureCodingPolicy() != null) {
    builder.setEcPolicy(convertErasureCodingPolicy(
        lb.getErasureCodingPolicy()));
  }
  return builder.setFileLength(lb.getFileLength())
      .setUnderConstruction(lb.isUnderConstruction())
      .addAllBlocks(convertLocatedBlocks2(lb.getLocatedBlocks()))
      .setIsLastBlockComplete(lb.isLastBlockComplete()).build();
}
 

import org.apache.hadoop.hdfs.protocol.LocatedBlocks; //导入方法依赖的package包/类
/**
 * This test writes a file and gets the block locations without closing the
 * file, and tests the block token in the last block. Block token is verified
 * by ensuring it is of correct kind.
 * 
 * @throws IOException
 * @throws InterruptedException
 */
@Test
public void testBlockTokenInLastLocatedBlock() throws IOException,
    InterruptedException {
  Configuration conf = new HdfsConfiguration();
  conf.setBoolean(DFSConfigKeys.DFS_BLOCK_ACCESS_TOKEN_ENABLE_KEY, true);
  conf.setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 512);
  MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
      .numDataNodes(1).build();
  cluster.waitActive();

  try {
    FileSystem fs = cluster.getFileSystem();
    String fileName = "/testBlockTokenInLastLocatedBlock";
    Path filePath = new Path(fileName);
    FSDataOutputStream out = fs.create(filePath, (short) 1);
    out.write(new byte[1000]);
    // ensure that the first block is written out (see FSOutputSummer#flush)
    out.flush();
    LocatedBlocks locatedBlocks = cluster.getNameNodeRpc().getBlockLocations(
        fileName, 0, 1000);
    while (locatedBlocks.getLastLocatedBlock() == null) {
      Thread.sleep(100);
      locatedBlocks = cluster.getNameNodeRpc().getBlockLocations(fileName, 0,
          1000);
    }
    Token<BlockTokenIdentifier> token = locatedBlocks.getLastLocatedBlock()
        .getBlockToken();
    Assert.assertEquals(BlockTokenIdentifier.KIND_NAME, token.getKind());
    out.close();
  } finally {
    cluster.shutdown();
  }
}
 

import org.apache.hadoop.hdfs.protocol.LocatedBlocks; //导入方法依赖的package包/类
@Test
/** Abandon a block while creating a file */
public void testAbandonBlock() throws IOException {
  String src = FILE_NAME_PREFIX + "foo";

  // Start writing a file but do not close it
  FSDataOutputStream fout = fs.create(new Path(src), true, 4096, (short)1, 512L);
  for (int i = 0; i < 1024; i++) {
    fout.write(123);
  }
  fout.hflush();
  long fileId = ((DFSOutputStream)fout.getWrappedStream()).getFileId();

  // Now abandon the last block
  DFSClient dfsclient = DFSClientAdapter.getDFSClient(fs);
  LocatedBlocks blocks =
    dfsclient.getNamenode().getBlockLocations(src, 0, Integer.MAX_VALUE);
  int orginalNumBlocks = blocks.locatedBlockCount();
  LocatedBlock b = blocks.getLastLocatedBlock();
  dfsclient.getNamenode().abandonBlock(b.getBlock(), fileId, src,
      dfsclient.clientName);
  
  // call abandonBlock again to make sure the operation is idempotent
  dfsclient.getNamenode().abandonBlock(b.getBlock(), fileId, src,
      dfsclient.clientName);

  // And close the file
  fout.close();

  // Close cluster and check the block has been abandoned after restart
  cluster.restartNameNode();
  blocks = dfsclient.getNamenode().getBlockLocations(src, 0,
      Integer.MAX_VALUE);
  Assert.assertEquals("Blocks " + b + " has not been abandoned.",
      orginalNumBlocks, blocks.locatedBlockCount() + 1);
}
 

import org.apache.hadoop.hdfs.protocol.LocatedBlocks; //导入方法依赖的package包/类
private LocatedBlocks waitForRecoveryFinished(Path file, int groupSize) 
    throws Exception {
  final int ATTEMPTS = 60;
  for (int i = 0; i < ATTEMPTS; i++) {
    LocatedBlocks locatedBlocks = getLocatedBlocks(file);
    LocatedStripedBlock lastBlock = 
        (LocatedStripedBlock)locatedBlocks.getLastLocatedBlock();
    DatanodeInfo[] storageInfos = lastBlock.getLocations();
    if (storageInfos.length >= groupSize) {
      return locatedBlocks;
    }
    Thread.sleep(1000);
  }
  throw new IOException ("Time out waiting for EC block recovery.");
}
 

import org.apache.hadoop.hdfs.protocol.LocatedBlocks; //导入方法依赖的package包/类
private void corruptBlocks(Path srcPath, int dataBlkDelNum,
    int parityBlkDelNum, boolean deleteBlockFile) throws IOException {
  int recoverBlkNum = dataBlkDelNum + parityBlkDelNum;

  LocatedBlocks locatedBlocks = getLocatedBlocks(srcPath);
  LocatedStripedBlock lastBlock =
      (LocatedStripedBlock)locatedBlocks.getLastLocatedBlock();

  int[] delDataBlkIndices = StripedFileTestUtil.randomArray(0, dataBlocks,
      dataBlkDelNum);
  Assert.assertNotNull(delDataBlkIndices);
  int[] delParityBlkIndices = StripedFileTestUtil.randomArray(dataBlocks,
      dataBlocks + parityBlocks, parityBlkDelNum);
  Assert.assertNotNull(delParityBlkIndices);

  int[] delBlkIndices = new int[recoverBlkNum];
  System.arraycopy(delDataBlkIndices, 0,
      delBlkIndices, 0, delDataBlkIndices.length);
  System.arraycopy(delParityBlkIndices, 0,
      delBlkIndices, delDataBlkIndices.length, delParityBlkIndices.length);

  ExtendedBlock[] delBlocks = new ExtendedBlock[recoverBlkNum];
  for (int i = 0; i < recoverBlkNum; i++) {
    delBlocks[i] = StripedBlockUtil
        .constructInternalBlock(lastBlock.getBlock(),
            cellSize, dataBlocks, delBlkIndices[i]);
    if (deleteBlockFile) {
      // delete the block file
      cluster.corruptBlockOnDataNodesByDeletingBlockFile(delBlocks[i]);
    } else {
      // corrupt the block file
      cluster.corruptBlockOnDataNodes(delBlocks[i]);
    }
  }
}