Java HftpFileSystem类代码示例

import org.apache.hadoop.hdfs.HftpFileSystem; //导入依赖的package包/类
void testFileCrcInternal(boolean inlineChecksum) throws IOException {
  ((Log4JLogger) HftpFileSystem.LOG).getLogger().setLevel(Level.ALL);

  Random random = new Random(1);

  final long seed = random.nextLong();
  random.setSeed(seed);
  FileSystem fs = FileSystem.getLocal(new Configuration());

  // generate random data
  final byte[] data = new byte[1024 * 1024 + 512 * 7 + 66];
  random.nextBytes(data);

  // write data to a file
  Path foo = new Path(TEST_ROOT_DIR, "foo_" + inlineChecksum);
  {
    final FSDataOutputStream out = fs.create(foo, false, 512, (short) 2, 512);
    out.write(data);
    out.close();
  }

  // compute data CRC
  DataChecksum checksum = DataChecksum.newDataChecksum(
      DataChecksum.CHECKSUM_CRC32, 1);
  checksum.update(data, 0, data.length);

  // compute checksum
  final int crc = fs.getFileCrc(foo);
  System.out.println("crc=" + crc);

  TestCase.assertEquals((int) checksum.getValue(), crc);
}
 

import org.apache.hadoop.hdfs.HftpFileSystem; //导入依赖的package包/类
static public Credentials getDTfromRemote(String nnAddr, 
    String renewer) throws IOException {
  DataInputStream dis = null;
  InetSocketAddress serviceAddr = NetUtils.createSocketAddr(nnAddr);
  
  try {
    StringBuffer url = new StringBuffer();
    if (renewer != null) {
      url.append(nnAddr).append(GetDelegationTokenServlet.PATH_SPEC)
         .append("?").append(GetDelegationTokenServlet.RENEWER).append("=")
         .append(renewer);
    } else {
      url.append(nnAddr).append(GetDelegationTokenServlet.PATH_SPEC);
    }
    URL remoteURL = new URL(url.toString());
    URLConnection connection = SecurityUtil.openSecureHttpConnection(remoteURL);
    InputStream in = connection.getInputStream();
    Credentials ts = new Credentials();
    dis = new DataInputStream(in);
    ts.readFields(dis);
    for(Token<?> token: ts.getAllTokens()) {
      token.setKind(HftpFileSystem.TOKEN_KIND);
      SecurityUtil.setTokenService(token, serviceAddr);
    }
    return ts;
  } catch (Exception e) {
    throw new IOException("Unable to obtain remote token", e);
  } finally {
    if(dis != null) dis.close();
  }
}
 

import org.apache.hadoop.hdfs.HftpFileSystem; //导入依赖的package包/类
@Test
public void testLazyRenewerStartup() {
  DelegationTokenRenewer<HftpFileSystem> dtr =
      new DelegationTokenRenewer<HftpFileSystem>(HftpFileSystem.class);
  assertFalse(dtr.isAlive());
  dtr.start();
  assertFalse(dtr.isAlive());
  dtr.addRenewAction(null);
  assertTrue(dtr.isAlive());
}
 

import org.apache.hadoop.hdfs.HftpFileSystem; //导入依赖的package包/类
protected SimpleDateFormat initialValue() {
  return HftpFileSystem.getDateFormat();
}
 

import org.apache.hadoop.hdfs.HftpFileSystem; //导入依赖的package包/类
@Override
protected SimpleDateFormat initialValue() {
  return HftpFileSystem.getDateFormat();
}
 

import org.apache.hadoop.hdfs.HftpFileSystem; //导入依赖的package包/类
/**
 * Utility method to obtain a delegation token over http
 * @param nnAddr Namenode http addr, such as http://namenode:50070
 * @param renewer User that is renewing the ticket in such a request
 */
static public Credentials getDTfromRemote(String nnAddr, 
                                          String renewer) throws IOException {
  DataInputStream dis = null;
  InetSocketAddress serviceAddr = NetUtils.createSocketAddr(nnAddr);

  try {
    StringBuffer url = new StringBuffer();
    if (renewer != null) {
      url.append(nnAddr).append(GetDelegationTokenServlet.PATH_SPEC).append("?").
      append(GetDelegationTokenServlet.RENEWER).append("=").append(renewer);
    } else {
      url.append(nnAddr).append(GetDelegationTokenServlet.PATH_SPEC);
    }
    if(LOG.isDebugEnabled()) {
      LOG.debug("Retrieving token from: " + url);
    }
    URL remoteURL = new URL(url.toString());
    boolean isSecure = "https".equals(remoteURL.getProtocol().toLowerCase());
    URLConnection connection = 
     SecurityUtil.openSecureHttpConnection(remoteURL);

    InputStream in = connection.getInputStream();
    Credentials ts = new Credentials();
    dis = new DataInputStream(in);
    ts.readFields(dis);
    for(Token<?> token: ts.getAllTokens()) {
      if (isSecure) {
        token.setKind(HsftpFileSystem.TOKEN_KIND);
      } else {
        token.setKind(HftpFileSystem.TOKEN_KIND);
      }
      SecurityUtil.setTokenService(token, serviceAddr);
    }
    return ts;
  } catch (Exception e) {
    throw new IOException("Unable to obtain remote token", e);
  } finally {
    if(dis != null) dis.close();
  }
}