Java HdfsDataInputStream.close方法代码示例

import org.apache.hadoop.hdfs.client.HdfsDataInputStream; //导入方法依赖的package包/类
static void checkFile(Path p, int expectedsize, final Configuration conf
    ) throws IOException, InterruptedException {
  //open the file with another user account
  final String username = UserGroupInformation.getCurrentUser().getShortUserName()
      + "_" + ++userCount;

  UserGroupInformation ugi = UserGroupInformation.createUserForTesting(username, 
                               new String[] {"supergroup"});
  
  final FileSystem fs = DFSTestUtil.getFileSystemAs(ugi, conf);
  
  final HdfsDataInputStream in = (HdfsDataInputStream)fs.open(p);

  //Check visible length
  Assert.assertTrue(in.getVisibleLength() >= expectedsize);

  //Able to read?
  for(int i = 0; i < expectedsize; i++) {
    Assert.assertEquals((byte)i, (byte)in.read());  
  }

  in.close();
}
 

import org.apache.hadoop.hdfs.client.HdfsDataInputStream; //导入方法依赖的package包/类
void zeroCopyRead(FileSystem fs,String path,
   int readSize,int nloop) throws IOException{

   long start_ts,end_ts,len=0;

   ByteBuffer bb = ByteBuffer.allocate(readSize);
   HdfsDataInputStream fsis = (HdfsDataInputStream)fs.open(new Path(path));
   for(int i=0;i<nloop;i++){
     fsis.seek(0);
     len=0;
     long ts1,ts2;
     start_ts = System.nanoTime();
     while(true){
bb = fsis.rdmaRead(readSize);
       if(bb==null)break;
       len += bb.remaining();
     };
     end_ts = System.nanoTime();
     System.out.println(((double)len/(end_ts - start_ts))+" GB/s");
   }
   fsis.close();
 }
 

import org.apache.hadoop.hdfs.client.HdfsDataInputStream; //导入方法依赖的package包/类
static void checkFile(Path p, int expectedsize, final Configuration conf)
    throws IOException, InterruptedException {
  //open the file with another user account
  final String username =
      UserGroupInformation.getCurrentUser().getShortUserName() + "_" +
          ++userCount;

  UserGroupInformation ugi = UserGroupInformation
      .createUserForTesting(username, new String[]{"supergroup"});
  
  final FileSystem fs = DFSTestUtil.getFileSystemAs(ugi, conf);
  
  final HdfsDataInputStream in = (HdfsDataInputStream) fs.open(p);

  //Check visible length
  Assert.assertTrue(in.getVisibleLength() >= expectedsize);

  //Able to read?
  for (int i = 0; i < expectedsize; i++) {
    Assert.assertEquals((byte) i, (byte) in.read());
  }

  in.close();
}
 

import org.apache.hadoop.hdfs.client.HdfsDataInputStream; //导入方法依赖的package包/类
public static ExtendedBlock getFirstBlock(FileSystem fs, Path path) throws IOException {
  HdfsDataInputStream in = (HdfsDataInputStream) fs.open(path);
  try {
    in.readByte();
    return in.getCurrentBlock();
  } finally {
    in.close();
  }
}
 

import org.apache.hadoop.hdfs.client.HdfsDataInputStream; //导入方法依赖的package包/类
/**
 * Read in-memory block with Short Circuit Read
 * Note: the test uses faked RAM_DISK from physical disk.
 */
@Test
public void testRamDiskShortCircuitRead()
    throws IOException, InterruptedException, TimeoutException {
  getClusterBuilder().setUseScr(true).build();
  final String METHOD_NAME = GenericTestUtils.getMethodName();
  final int SEED = 0xFADED;
  Path path = new Path("/" + METHOD_NAME + ".dat");

  // Create a file and wait till it is persisted.
  makeRandomTestFile(path, BLOCK_SIZE, true, SEED);
  ensureFileReplicasOnStorageType(path, RAM_DISK);
  waitForMetric("RamDiskBlocksLazyPersisted", 1);

  HdfsDataInputStream fis = (HdfsDataInputStream) fs.open(path);

  // Verify SCR read counters
  try {
    byte[] buf = new byte[BUFFER_LENGTH];
    fis.read(0, buf, 0, BUFFER_LENGTH);
    Assert.assertEquals(BUFFER_LENGTH,
      fis.getReadStatistics().getTotalBytesRead());
    Assert.assertEquals(BUFFER_LENGTH,
      fis.getReadStatistics().getTotalShortCircuitBytesRead());
  } finally {
    fis.close();
    fis = null;
  }
}
 

import org.apache.hadoop.hdfs.client.HdfsDataInputStream; //导入方法依赖的package包/类
/** Check the file content, reading as user {@code readingUser} */
static void checkFileContentDirect(URI uri, Path name, byte[] expected,
    int readOffset, String readingUser, Configuration conf,
    boolean legacyShortCircuitFails)
    throws IOException, InterruptedException {
  // Ensure short circuit is enabled
  DistributedFileSystem fs = getFileSystem(readingUser, uri, conf);
  ClientContext clientContext = ClientContext.getFromConf(conf);
  if (legacyShortCircuitFails) {
    assertTrue(clientContext.getDisableLegacyBlockReaderLocal());
  }
  
  HdfsDataInputStream stm = (HdfsDataInputStream)fs.open(name);

  ByteBuffer actual = ByteBuffer.allocateDirect(expected.length - readOffset);

  IOUtils.skipFully(stm, readOffset);

  actual.limit(3);

  //Read a small number of bytes first.
  int nread = stm.read(actual);
  actual.limit(nread + 2);
  nread += stm.read(actual);

  // Read across chunk boundary
  actual.limit(Math.min(actual.capacity(), nread + 517));
  nread += stm.read(actual);
  checkData(arrayFromByteBuffer(actual), readOffset, expected, nread,
      "A few bytes");
  //Now read rest of it
  actual.limit(actual.capacity());
  while (actual.hasRemaining()) {
    int nbytes = stm.read(actual);

    if (nbytes < 0) {
      throw new EOFException("End of file reached before reading fully.");
    }
    nread += nbytes;
  }
  checkData(arrayFromByteBuffer(actual), readOffset, expected, "Read 3");
  if (legacyShortCircuitFails) {
    assertTrue(clientContext.getDisableLegacyBlockReaderLocal());
  }
  stm.close();
}
 

import org.apache.hadoop.hdfs.client.HdfsDataInputStream; //导入方法依赖的package包/类
/** Check the file content, reading as user {@code readingUser} */
static void checkFileContentDirect(URI uri, Path name, byte[] expected,
    int readOffset, String readingUser, Configuration conf,
    boolean legacyShortCircuitFails)
    throws IOException, InterruptedException {
  // Ensure short circuit is enabled
  DistributedFileSystem fs = getFileSystem(readingUser, uri, conf);
  ClientContext clientContext = ClientContext.getFromConf(conf);
  if (legacyShortCircuitFails) {
    assertTrue(clientContext.getDisableLegacyBlockReaderLocal());
  }

  HdfsDataInputStream stm = (HdfsDataInputStream)fs.open(name);

  ByteBuffer actual = ByteBuffer.allocateDirect(expected.length - readOffset);

  IOUtils.skipFully(stm, readOffset);

  actual.limit(3);

  //Read a small number of bytes first.
  int nread = stm.read(actual);
  actual.limit(nread + 2);
  nread += stm.read(actual);

  // Read across chunk boundary
  actual.limit(Math.min(actual.capacity(), nread + 517));
  nread += stm.read(actual);
  checkData(arrayFromByteBuffer(actual), readOffset, expected, nread,
      "A few bytes");
  //Now read rest of it
  actual.limit(actual.capacity());
  while (actual.hasRemaining()) {
    int nbytes = stm.read(actual);

    if (nbytes < 0) {
      throw new EOFException("End of file reached before reading fully.");
    }
    nread += nbytes;
  }
  checkData(arrayFromByteBuffer(actual), readOffset, expected, "Read 3");
  if (legacyShortCircuitFails) {
    assertTrue(clientContext.getDisableLegacyBlockReaderLocal());
  }
  stm.close();
}
 

import org.apache.hadoop.hdfs.client.HdfsDataInputStream; //导入方法依赖的package包/类
@Test
public void testExternalBlockReader() throws Exception {
  Configuration conf = new Configuration();
  conf.set(HdfsClientConfigKeys.REPLICA_ACCESSOR_BUILDER_CLASSES_KEY,
      SyntheticReplicaAccessorBuilder.class.getName());
  conf.setLong(HdfsClientConfigKeys.DFS_BLOCK_SIZE_KEY, 1024);
  conf.setLong(DFSConfigKeys.DFS_NAMENODE_MIN_BLOCK_SIZE_KEY, 0);
  String uuid = UUID.randomUUID().toString();
  conf.set(SYNTHETIC_BLOCK_READER_TEST_UUID_KEY, uuid);
  MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
      .numDataNodes(1)
      .build();
  final int TEST_LENGTH = 2047;
  DistributedFileSystem dfs = cluster.getFileSystem();
  try {
    DFSTestUtil.createFile(dfs, new Path("/a"), TEST_LENGTH, (short)1, SEED);
    HdfsDataInputStream stream =
        (HdfsDataInputStream)dfs.open(new Path("/a"));
    byte buf[] = new byte[TEST_LENGTH];
    stream.seek(1000);
    IOUtils.readFully(stream, buf, 1000, TEST_LENGTH - 1000);
    stream.seek(0);
    IOUtils.readFully(stream, buf, 0, 1000);
    byte expected[] = DFSTestUtil.
        calculateFileContentsFromSeed(SEED, TEST_LENGTH);
    ReadStatistics stats = stream.getReadStatistics();
    Assert.assertEquals(1024, stats.getTotalShortCircuitBytesRead());
    Assert.assertEquals(2047, stats.getTotalLocalBytesRead());
    Assert.assertEquals(2047, stats.getTotalBytesRead());
    Assert.assertArrayEquals(expected, buf);
    stream.close();
    ExtendedBlock block = DFSTestUtil.getFirstBlock(dfs, new Path("/a"));
    Assert.assertNotNull(block);
    LinkedList<SyntheticReplicaAccessor> accessorList = accessors.get(uuid);
    Assert.assertNotNull(accessorList);
    Assert.assertEquals(3, accessorList.size());
    SyntheticReplicaAccessor accessor = accessorList.get(0);
    Assert.assertTrue(accessor.builder.allowShortCircuit);
    Assert.assertEquals(block.getBlockPoolId(),
        accessor.builder.blockPoolId);
    Assert.assertEquals(block.getBlockId(),
        accessor.builder.blockId);
    Assert.assertEquals(dfs.getClient().clientName,
        accessor.builder.clientName);
    Assert.assertEquals("/a", accessor.builder.fileName);
    Assert.assertEquals(block.getGenerationStamp(),
        accessor.getGenerationStamp());
    Assert.assertTrue(accessor.builder.verifyChecksum);
    Assert.assertEquals(1024L, accessor.builder.visibleLength);
    Assert.assertEquals(24L, accessor.totalRead);
    Assert.assertEquals("", accessor.getError());
    Assert.assertEquals(1, accessor.numCloses);
    byte[] tempBuf = new byte[5];
    Assert.assertEquals(-1, accessor.read(TEST_LENGTH,
          tempBuf, 0, 0));
    Assert.assertEquals(-1, accessor.read(TEST_LENGTH,
          tempBuf, 0, tempBuf.length));
    accessors.remove(uuid);
  } finally {
    dfs.close();
    cluster.shutdown();
  }
}
 

import org.apache.hadoop.hdfs.client.HdfsDataInputStream; //导入方法依赖的package包/类
/** Check the file content, reading as user {@code readingUser} */
static void checkFileContentDirect(URI uri, Path name, byte[] expected,
    int readOffset, String readingUser, Configuration conf,
    boolean legacyShortCircuitFails)
    throws IOException, InterruptedException {
  // Ensure short circuit is enabled
  DistributedFileSystem fs = getFileSystem(readingUser, uri, conf);
  if (legacyShortCircuitFails) {
    assertTrue(fs.getClient().useLegacyBlockReaderLocal());
  }
  
  HdfsDataInputStream stm = (HdfsDataInputStream)fs.open(name);

  ByteBuffer actual = ByteBuffer.allocateDirect(expected.length - readOffset);

  IOUtils.skipFully(stm, readOffset);

  actual.limit(3);

  //Read a small number of bytes first.
  int nread = stm.read(actual);
  actual.limit(nread + 2);
  nread += stm.read(actual);

  // Read across chunk boundary
  actual.limit(Math.min(actual.capacity(), nread + 517));
  nread += stm.read(actual);
  checkData(arrayFromByteBuffer(actual), readOffset, expected, nread,
      "A few bytes");
  //Now read rest of it
  actual.limit(actual.capacity());
  while (actual.hasRemaining()) {
    int nbytes = stm.read(actual);

    if (nbytes < 0) {
      throw new EOFException("End of file reached before reading fully.");
    }
    nread += nbytes;
  }
  checkData(arrayFromByteBuffer(actual), readOffset, expected, "Read 3");
  if (legacyShortCircuitFails) {
    assertFalse(fs.getClient().useLegacyBlockReaderLocal());
  }
  stm.close();
}