Java ClientProtocol.getBlockLocations方法代码示例

import org.apache.hadoop.hdfs.protocol.ClientProtocol; //导入方法依赖的package包/类
/**
 * @see ClientProtocol#getBlockLocations(String, long, long)
 */
static LocatedBlocks callGetBlockLocations(ClientProtocol namenode,
    String src, long start, long length) 
    throws IOException {
  try {
    return namenode.getBlockLocations(src, start, length);
  } catch(RemoteException re) {
    throw re.unwrapRemoteException(AccessControlException.class,
                                   FileNotFoundException.class,
                                   UnresolvedPathException.class);
  }
}
 

import org.apache.hadoop.hdfs.protocol.ClientProtocol; //导入方法依赖的package包/类
/** Create a redirection URL */
private URL createRedirectURL(String path, String encodedPath, HdfsFileStatus status, 
    UserGroupInformation ugi, ClientProtocol nnproxy, HttpServletRequest request, String dt)
    throws IOException {
  String scheme = request.getScheme();
  final LocatedBlocks blks = nnproxy.getBlockLocations(
      status.getFullPath(new Path(path)).toUri().getPath(), 0, 1);
  final Configuration conf = NameNodeHttpServer.getConfFromContext(
      getServletContext());
  final DatanodeID host = pickSrcDatanode(blks, status, conf);
  final String hostname;
  if (host instanceof DatanodeInfo) {
    hostname = host.getHostName();
  } else {
    hostname = host.getIpAddr();
  }

  int port = "https".equals(scheme) ? host.getInfoSecurePort() : host
      .getInfoPort();

  String dtParam = "";
  if (dt != null) {
    dtParam = JspHelper.getDelegationTokenUrlParam(dt);
  }

  // Add namenode address to the url params
  NameNode nn = NameNodeHttpServer.getNameNodeFromContext(
      getServletContext());
  String addr = nn.getNameNodeAddressHostPortString();
  String addrParam = JspHelper.getUrlParam(JspHelper.NAMENODE_ADDRESS, addr);
  
  return new URL(scheme, hostname, port,
      "/streamFile" + encodedPath + '?' +
      "ugi=" + ServletUtil.encodeQueryValue(ugi.getShortUserName()) +
      dtParam + addrParam);
}
 

import org.apache.hadoop.hdfs.protocol.ClientProtocol; //导入方法依赖的package包/类
private void waitForBlockReplication(String filename, 
                                     ClientProtocol namenode,
                                     int expected, long maxWaitSec) 
                                     throws IOException {
  long start = Time.monotonicNow();
  
  //wait for all the blocks to be replicated;
  LOG.info("Checking for block replication for " + filename);
  
  LocatedBlocks blocks = namenode.getBlockLocations(filename, 0, Long.MAX_VALUE);
  assertEquals(numBlocks, blocks.locatedBlockCount());
  
  for (int i = 0; i < numBlocks; ++i) {
    LOG.info("Checking for block:" + (i+1));
    while (true) { // Loop to check for block i (usually when 0 is done all will be done
      blocks = namenode.getBlockLocations(filename, 0, Long.MAX_VALUE);
      assertEquals(numBlocks, blocks.locatedBlockCount());
      LocatedBlock block = blocks.get(i);
      int actual = block.getLocations().length;
      if ( actual == expected ) {
        LOG.info("Got enough replicas for " + (i+1) + "th block " + block.getBlock() +
            ", got " + actual + ".");
        break;
      }
      LOG.info("Not enough replicas for " + (i+1) + "th block " + block.getBlock() +
                             " yet. Expecting " + expected + ", got " + 
                             actual + ".");
    
      if (maxWaitSec > 0 && 
          (Time.monotonicNow() - start) > (maxWaitSec * 1000)) {
        throw new IOException("Timedout while waiting for all blocks to " +
                              " be replicated for " + filename);
      }
    
      try {
        Thread.sleep(500);
      } catch (InterruptedException ignored) {}
    }
  }
}
 

import org.apache.hadoop.hdfs.protocol.ClientProtocol; //导入方法依赖的package包/类
public static LocatedBlock getLastLocatedBlock(
    ClientProtocol namenode, String src) throws IOException {
  //get block info for the last block
  LocatedBlocks locations = namenode.getBlockLocations(src, 0, Long.MAX_VALUE);
  List<LocatedBlock> blocks = locations.getLocatedBlocks();
  DataNode.LOG.info("blocks.size()=" + blocks.size());
  assertTrue(blocks.size() > 0);

  return blocks.get(blocks.size() - 1);
}
 

import org.apache.hadoop.hdfs.protocol.ClientProtocol; //导入方法依赖的package包/类
private void waitForBlockReplication(String filename, 
                                     ClientProtocol namenode,
                                     int expected, long maxWaitSec) 
                                     throws IOException {
  long start = Time.monotonicNow();
  
  //wait for all the blocks to be replicated;
  LOG.info("Checking for block replication for " + filename);
  while (true) {
    boolean replOk = true;
    LocatedBlocks blocks = namenode.getBlockLocations(filename, 0, 
                                                      Long.MAX_VALUE);
    
    for (Iterator<LocatedBlock> iter = blocks.getLocatedBlocks().iterator();
         iter.hasNext();) {
      LocatedBlock block = iter.next();
      int actual = block.getLocations().length;
      if ( actual < expected ) {
        LOG.info("Not enough replicas for " + block.getBlock()
            + " yet. Expecting " + expected + ", got " + actual + ".");
        replOk = false;
        break;
      }
    }
    
    if (replOk) {
      return;
    }
    
    if (maxWaitSec > 0 && 
        (Time.monotonicNow() - start) > (maxWaitSec * 1000)) {
      throw new IOException("Timedout while waiting for all blocks to " +
                            " be replicated for " + filename);
    }
    
    try {
      Thread.sleep(500);
    } catch (InterruptedException ignored) {}
  }
}
 

import org.apache.hadoop.hdfs.protocol.ClientProtocol; //导入方法依赖的package包/类
/**
 * @see ClientProtocol#getBlockLocations(String, long, long)
 */
static LocatedBlocks callGetBlockLocations(ClientProtocol namenode,
    String src, long start, long length)
    throws IOException {
  try {
    return namenode.getBlockLocations(src, start, length);
  } catch(RemoteException re) {
    throw re.unwrapRemoteException(AccessControlException.class,
        FileNotFoundException.class,
        UnresolvedPathException.class);
  }
}
 

import org.apache.hadoop.hdfs.protocol.ClientProtocol; //导入方法依赖的package包/类
private void waitForBlockReplication(String filename,
    ClientProtocol namenode,
    int expected, long maxWaitSec,
    boolean isUnderConstruction, boolean noOverReplication)
    throws IOException {
  long start = Time.monotonicNow();
  
  //wait for all the blocks to be replicated;
  LOG.info("Checking for block replication for " + filename);
  while (true) {
    boolean replOk = true;
    LocatedBlocks blocks = namenode.getBlockLocations(filename, 0, 
                                                      Long.MAX_VALUE);
    
    for (Iterator<LocatedBlock> iter = blocks.getLocatedBlocks().iterator();
         iter.hasNext();) {
      LocatedBlock block = iter.next();
      if (isUnderConstruction && !iter.hasNext()) {
        break; // do not check the last block
      }
      int actual = block.getLocations().length;
      if (noOverReplication) {
        assertTrue(actual <= expected);
      }
      if ( actual < expected ) {
        LOG.info("Not enough replicas for " + block.getBlock()
            + " yet. Expecting " + expected + ", got " + actual + ".");
        replOk = false;
        break;
      }
    }
    
    if (replOk) {
      return;
    }
    
    if (maxWaitSec > 0 && 
        (Time.monotonicNow() - start) > (maxWaitSec * 1000)) {
      throw new IOException("Timedout while waiting for all blocks to " +
                            " be replicated for " + filename);
    }
    
    try {
      Thread.sleep(500);
    } catch (InterruptedException ignored) {}
  }
}