Java ClientDatanodeProtocol.getBlockLocalPathInfo方法代码示例

import org.apache.hadoop.hdfs.protocol.ClientDatanodeProtocol; //导入方法依赖的package包/类
private static BlockLocalPathInfo getBlockPathInfo(UserGroupInformation ugi,
    ExtendedBlock blk, DatanodeInfo node, Configuration conf, int timeout,
    Token<BlockTokenIdentifier> token, boolean connectToDnViaHostname)
    throws IOException {
  LocalDatanodeInfo localDatanodeInfo = getLocalDatanodeInfo(node.getIpcPort());
  BlockLocalPathInfo pathinfo = null;
  ClientDatanodeProtocol proxy = localDatanodeInfo.getDatanodeProxy(ugi, node,
      conf, timeout, connectToDnViaHostname);
  try {
    // make RPC to local datanode to find local pathnames of blocks
    pathinfo = proxy.getBlockLocalPathInfo(blk, token);
    if (pathinfo != null) {
      if (LOG.isDebugEnabled()) {
        LOG.debug("Cached location of block " + blk + " as " + pathinfo);
      }
      localDatanodeInfo.setBlockLocalPathInfo(blk, pathinfo);
    }
  } catch (IOException e) {
    localDatanodeInfo.resetDatanodeProxy(); // Reset proxy on error
    throw e;
  }
  return pathinfo;
}
 

import org.apache.hadoop.hdfs.protocol.ClientDatanodeProtocol; //导入方法依赖的package包/类
private static BlockLocalPathInfo getBlockPathInfo(ExtendedBlock blk,
    DatanodeInfo node, Configuration conf, int timeout,
    Token<BlockTokenIdentifier> token, boolean connectToDnViaHostname)
    throws IOException {
  LocalDatanodeInfo localDatanodeInfo =
      getLocalDatanodeInfo(node.getIpcPort());
  BlockLocalPathInfo pathinfo = null;
  ClientDatanodeProtocol proxy = localDatanodeInfo
      .getDatanodeProxy(node, conf, timeout, connectToDnViaHostname);
  try {
    // make RPC to local datanode to find local pathnames of blocks
    pathinfo = proxy.getBlockLocalPathInfo(blk, token);
    if (pathinfo != null) {
      if (LOG.isDebugEnabled()) {
        LOG.debug("Cached location of block " + blk + " as " + pathinfo);
      }
      localDatanodeInfo.setBlockLocalPathInfo(blk, pathinfo);
    }
  } catch (IOException e) {
    localDatanodeInfo.resetDatanodeProxy(); // Reset proxy on error
    throw e;
  }
  return pathinfo;
}
 

import org.apache.hadoop.hdfs.protocol.ClientDatanodeProtocol; //导入方法依赖的package包/类
private static BlockLocalPathInfo getBlockPathInfo(Block blk,
    DatanodeInfo node, Configuration conf, int timeout,
    Token<BlockTokenIdentifier> token) throws IOException {
  LocalDatanodeInfo localDatanodeInfo = getLocalDatanodeInfo(node.ipcPort);
  BlockLocalPathInfo pathinfo = null;
  ClientDatanodeProtocol proxy = localDatanodeInfo.getDatanodeProxy(node,
      conf, timeout);
  try {
    // make RPC to local datanode to find local pathnames of blocks
    pathinfo = proxy.getBlockLocalPathInfo(blk, token);
    if (pathinfo != null) {
      if (LOG.isDebugEnabled()) {
        LOG.debug("Cached location of block " + blk + " as " + pathinfo);
      }
      localDatanodeInfo.setBlockLocalPathInfo(blk, pathinfo);
    }
  } catch (IOException e) {
    localDatanodeInfo.resetDatanodeProxy(); // Reset proxy on error
    throw e;
  }
  return pathinfo;
}
 

import org.apache.hadoop.hdfs.protocol.ClientDatanodeProtocol; //导入方法依赖的package包/类
private static BlockLocalPathInfo getBlockPathInfo(UserGroupInformation ugi,
    ExtendedBlock blk, DatanodeInfo node, Configuration conf, int timeout,
    Token<BlockTokenIdentifier> token, boolean connectToDnViaHostname,
    StorageType storageType) throws IOException {
  LocalDatanodeInfo localDatanodeInfo = getLocalDatanodeInfo(node.getIpcPort());
  BlockLocalPathInfo pathinfo = null;
  ClientDatanodeProtocol proxy = localDatanodeInfo.getDatanodeProxy(ugi, node,
      conf, timeout, connectToDnViaHostname);
  try {
    // make RPC to local datanode to find local pathnames of blocks
    pathinfo = proxy.getBlockLocalPathInfo(blk, token);
    // We cannot cache the path information for a replica on transient storage.
    // If the replica gets evicted, then it moves to a different path.  Then,
    // our next attempt to read from the cached path would fail to find the
    // file.  Additionally, the failure would cause us to disable legacy
    // short-circuit read for all subsequent use in the ClientContext.  Unlike
    // the newer short-circuit read implementation, we have no communication
    // channel for the DataNode to notify the client that the path has been
    // invalidated.  Therefore, our only option is to skip caching.
    if (pathinfo != null && !storageType.isTransient()) {
      if (LOG.isDebugEnabled()) {
        LOG.debug("Cached location of block " + blk + " as " + pathinfo);
      }
      localDatanodeInfo.setBlockLocalPathInfo(blk, pathinfo);
    }
  } catch (IOException e) {
    localDatanodeInfo.resetDatanodeProxy(); // Reset proxy on error
    throw e;
  }
  return pathinfo;
}
 

import org.apache.hadoop.hdfs.protocol.ClientDatanodeProtocol; //导入方法依赖的package包/类
@Test(timeout=10000)
public void testDeprecatedGetBlockLocalPathInfoRpc() throws IOException {
  final Configuration conf = new Configuration();
  MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1)
      .format(true).build();
  cluster.waitActive();
  FileSystem fs = cluster.getFileSystem();
  try {
    DFSTestUtil.createFile(fs, new Path("/tmp/x"), 16, (short) 1, 23);
    LocatedBlocks lb = cluster.getNameNode().getRpcServer()
        .getBlockLocations("/tmp/x", 0, 16);
    // Create a new block object, because the block inside LocatedBlock at
    // namenode is of type BlockInfo.
    ExtendedBlock blk = new ExtendedBlock(lb.get(0).getBlock());
    Token<BlockTokenIdentifier> token = lb.get(0).getBlockToken();
    final DatanodeInfo dnInfo = lb.get(0).getLocations()[0];
    ClientDatanodeProtocol proxy = 
        DFSUtil.createClientDatanodeProtocolProxy(dnInfo, conf, 60000, false);
    try {
      proxy.getBlockLocalPathInfo(blk, token);
      Assert.fail("The call should have failed as this user "
          + " is not allowed to call getBlockLocalPathInfo");
    } catch (IOException ex) {
      Assert.assertTrue(ex.getMessage().contains(
          "not allowed to call getBlockLocalPathInfo"));
    }
  } finally {
    fs.close();
    cluster.shutdown();
  }
}
 

import org.apache.hadoop.hdfs.protocol.ClientDatanodeProtocol; //导入方法依赖的package包/类
private static BlockLocalPathInfo getBlockPathInfo(UserGroupInformation ugi,
    ExtendedBlock blk, DatanodeInfo node, Configuration conf, int timeout,
    Token<BlockTokenIdentifier> token, boolean connectToDnViaHostname,
    StorageType storageType) throws IOException {
  LocalDatanodeInfo localDatanodeInfo =
      getLocalDatanodeInfo(node.getIpcPort());
  BlockLocalPathInfo pathinfo;
  ClientDatanodeProtocol proxy = localDatanodeInfo.getDatanodeProxy(ugi, node,
      conf, timeout, connectToDnViaHostname);
  try {
    // make RPC to local datanode to find local pathnames of blocks
    pathinfo = proxy.getBlockLocalPathInfo(blk, token);
    // We can't cache the path information for a replica on transient storage.
    // If the replica gets evicted, then it moves to a different path.  Then,
    // our next attempt to read from the cached path would fail to find the
    // file.  Additionally, the failure would cause us to disable legacy
    // short-circuit read for all subsequent use in the ClientContext.  Unlike
    // the newer short-circuit read implementation, we have no communication
    // channel for the DataNode to notify the client that the path has been
    // invalidated.  Therefore, our only option is to skip caching.
    if (pathinfo != null && !storageType.isTransient()) {
      LOG.debug("Cached location of block {} as {}", blk, pathinfo);
      localDatanodeInfo.setBlockLocalPathInfo(blk, pathinfo);
    }
  } catch (IOException e) {
    localDatanodeInfo.resetDatanodeProxy(); // Reset proxy on error
    throw e;
  }
  return pathinfo;
}
 

import org.apache.hadoop.hdfs.protocol.ClientDatanodeProtocol; //导入方法依赖的package包/类
@Test(timeout=10000)
public void testDeprecatedGetBlockLocalPathInfoRpc() throws IOException {
  final Configuration conf = new Configuration();
  MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1)
      .format(true).build();
  cluster.waitActive();
  FileSystem fs = cluster.getFileSystem();
  try {
    DFSTestUtil.createFile(fs, new Path("/tmp/x"), 16, (short) 1, 23);
    LocatedBlocks lb = cluster.getNameNode().getRpcServer()
        .getBlockLocations("/tmp/x", 0, 16);
    // Create a new block object, because the block inside LocatedBlock at
    // namenode is of type BlockInfo.
    ExtendedBlock blk = new ExtendedBlock(lb.get(0).getBlock());
    Token<BlockTokenIdentifier> token = lb.get(0).getBlockToken();
    final DatanodeInfo dnInfo = lb.get(0).getLocations()[0];
    ClientDatanodeProtocol proxy = 
        DFSUtilClient.createClientDatanodeProtocolProxy(dnInfo, conf, 60000, false);
    try {
      proxy.getBlockLocalPathInfo(blk, token);
      Assert.fail("The call should have failed as this user "
          + " is not allowed to call getBlockLocalPathInfo");
    } catch (IOException ex) {
      Assert.assertTrue(ex.getMessage().contains(
          "not allowed to call getBlockLocalPathInfo"));
    }
  } finally {
    fs.close();
    cluster.shutdown();
  }
}
 

import org.apache.hadoop.hdfs.protocol.ClientDatanodeProtocol; //导入方法依赖的package包/类
@Test(timeout=10000)
public void testDeprecatedGetBlockLocalPathInfoRpc()
    throws IOException, InterruptedException {
  final Configuration conf = new Configuration();
  MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1)
      .format(true).build();
  cluster.waitActive();
  FileSystem fs = cluster.getFileSystem();
  try {
    DFSTestUtil.createFile(fs, new Path("/tmp/x"), 16, (short) 1, 23);
    LocatedBlocks lb = cluster.getNameNode().getRpcServer()
        .getBlockLocations("/tmp/x", 0, 16);
    // Create a new block object, because the block inside LocatedBlock at
    // namenode is of type BlockInfo.
    ExtendedBlock blk = new ExtendedBlock(lb.get(0).getBlock());
    Token<BlockTokenIdentifier> token = lb.get(0).getBlockToken();
    final DatanodeInfo dnInfo = lb.get(0).getLocations()[0];
    ClientDatanodeProtocol proxy = 
        DFSUtil.createClientDatanodeProtocolProxy(dnInfo, conf, 60000, false);
    try {
      proxy.getBlockLocalPathInfo(blk, token);
      Assert.fail("The call should have failed as this user "
          + " is not allowed to call getBlockLocalPathInfo");
    } catch (IOException ex) {
      Assert.assertTrue(ex.getMessage().contains(
          "not allowed to call getBlockLocalPathInfo"));
    }
  } finally {
    fs.close();
    cluster.shutdown();
  }
}
 

import org.apache.hadoop.hdfs.protocol.ClientDatanodeProtocol; //导入方法依赖的package包/类
@Test
public void testGetBlockLocalPathInfo() throws IOException, InterruptedException {
  final Configuration conf = new Configuration();
  conf.set(DFSConfigKeys.DFS_BLOCK_LOCAL_PATH_ACCESS_USER_KEY, "alloweduser");
  MiniDFSCluster cluster = new MiniDFSCluster(conf, 1, true, null);
  cluster.waitActive();
  final DataNode dn = cluster.getDataNodes().get(0);
  FileSystem fs = cluster.getFileSystem();
  try {
    DFSTestUtil.createFile(fs, new Path("/tmp/x"), 16, (short) 1, 23);
    UserGroupInformation aUgi = UserGroupInformation
        .createRemoteUser("alloweduser");
    LocatedBlocks lb = cluster.getNameNode().getBlockLocations("/tmp/x", 0,
        16);
    // Create a new block object, because the block inside LocatedBlock at
    // namenode is of type BlockInfo.
    Block blk = new Block(lb.get(0).getBlock());
    Token<BlockTokenIdentifier> token = lb.get(0).getBlockToken();
    final DatanodeInfo dnInfo = lb.get(0).getLocations()[0];
    ClientDatanodeProtocol proxy = aUgi
        .doAs(new PrivilegedExceptionAction<ClientDatanodeProtocol>() {
          @Override
          public ClientDatanodeProtocol run() throws Exception {
            return DFSClient.createClientDatanodeProtocolProxy(
                dnInfo, conf, 60000);
          }
        });
    
    //This should succeed
    BlockLocalPathInfo blpi = proxy.getBlockLocalPathInfo(blk, token);
    Assert.assertEquals(dn.data.getBlockLocalPathInfo(blk).getBlockPath(),
        blpi.getBlockPath());
    RPC.stopProxy(proxy);

    // Now try with a not allowed user.
    UserGroupInformation bUgi = UserGroupInformation
        .createRemoteUser("notalloweduser");
    proxy = bUgi
        .doAs(new PrivilegedExceptionAction<ClientDatanodeProtocol>() {
          @Override
          public ClientDatanodeProtocol run() throws Exception {
            return DFSClient.createClientDatanodeProtocolProxy(
                dnInfo, conf, 60000);
          }
        });
    try {
      proxy.getBlockLocalPathInfo(blk, token);
      Assert.fail("The call should have failed as " + bUgi.getShortUserName()
          + " is not allowed to call getBlockLocalPathInfo");
    } catch (IOException ex) {
      Assert.assertTrue(ex.getMessage().contains(
          "not allowed to call getBlockLocalPathInfo"));
    } finally {
      RPC.stopProxy(proxy);
    }
  } finally {
    fs.close();
    cluster.shutdown();
  }
}