Java HdfsDataInputStream.read方法代码示例

import org.apache.hadoop.hdfs.client.HdfsDataInputStream; //导入方法依赖的package包/类
private boolean checkUnsupportedMethod(FileSystem fs, Path file,
                                         byte[] expected, int readOffset) throws IOException {
  HdfsDataInputStream stm = (HdfsDataInputStream)fs.open(file);
  ByteBuffer actual = ByteBuffer.allocateDirect(expected.length - readOffset);
  IOUtils.skipFully(stm, readOffset);
  try {
    stm.read(actual);
  } catch(UnsupportedOperationException unex) {
    return true;
  }
  return false;
}
 

import org.apache.hadoop.hdfs.client.HdfsDataInputStream; //导入方法依赖的package包/类
private boolean checkUnsupportedMethod(FileSystem fs, Path file,
    byte[] expected, int readOffset) throws IOException {
  HdfsDataInputStream stm = (HdfsDataInputStream)fs.open(file);
  ByteBuffer actual = ByteBuffer.allocateDirect(expected.length - readOffset);
  IOUtils.skipFully(stm, readOffset);
  try {
    stm.read(actual);
  } catch(UnsupportedOperationException unex) {
    return true;
  }
  return false;
}
 

import org.apache.hadoop.hdfs.client.HdfsDataInputStream; //导入方法依赖的package包/类
/**
 * Read in-memory block with Short Circuit Read
 * Note: the test uses faked RAM_DISK from physical disk.
 */
@Test
public void testRamDiskShortCircuitRead()
    throws IOException, InterruptedException, TimeoutException {
  getClusterBuilder().setUseScr(true).build();
  final String METHOD_NAME = GenericTestUtils.getMethodName();
  final int SEED = 0xFADED;
  Path path = new Path("/" + METHOD_NAME + ".dat");

  // Create a file and wait till it is persisted.
  makeRandomTestFile(path, BLOCK_SIZE, true, SEED);
  ensureFileReplicasOnStorageType(path, RAM_DISK);
  waitForMetric("RamDiskBlocksLazyPersisted", 1);

  HdfsDataInputStream fis = (HdfsDataInputStream) fs.open(path);

  // Verify SCR read counters
  try {
    byte[] buf = new byte[BUFFER_LENGTH];
    fis.read(0, buf, 0, BUFFER_LENGTH);
    Assert.assertEquals(BUFFER_LENGTH,
      fis.getReadStatistics().getTotalBytesRead());
    Assert.assertEquals(BUFFER_LENGTH,
      fis.getReadStatistics().getTotalShortCircuitBytesRead());
  } finally {
    fis.close();
    fis = null;
  }
}
 

import org.apache.hadoop.hdfs.client.HdfsDataInputStream; //导入方法依赖的package包/类
/**
 * Eviction of lazy persisted blocks with Short Circuit Read handle open
 * Note: the test uses faked RAM_DISK from physical disk.
 * @throws IOException
 * @throws InterruptedException
 */
@Test
public void tesScrDuringEviction()
    throws Exception {
  getClusterBuilder().setUseScr(true).build();
  final String METHOD_NAME = GenericTestUtils.getMethodName();
  Path path1 = new Path("/" + METHOD_NAME + ".01.dat");

  // Create a file and wait till it is persisted.
  makeTestFile(path1, BLOCK_SIZE, true);
  ensureFileReplicasOnStorageType(path1, RAM_DISK);
  waitForMetric("RamDiskBlocksLazyPersisted", 1);

  HdfsDataInputStream fis = (HdfsDataInputStream) fs.open(path1);
  try {
    // Keep and open read handle to path1 while creating path2
    byte[] buf = new byte[BUFFER_LENGTH];
    fis.read(0, buf, 0, BUFFER_LENGTH);
    triggerEviction(cluster.getDataNodes().get(0));

    // Ensure path1 is still readable from the open SCR handle.
    fis.read(0, buf, 0, BUFFER_LENGTH);
    assertThat(fis.getReadStatistics().getTotalBytesRead(),
        is((long) 2 * BUFFER_LENGTH));
    assertThat(fis.getReadStatistics().getTotalShortCircuitBytesRead(),
        is((long) 2 * BUFFER_LENGTH));
  } finally {
    IOUtils.closeQuietly(fis);
  }
}
 

import org.apache.hadoop.hdfs.client.HdfsDataInputStream; //导入方法依赖的package包/类
/** Check the file content, reading as user {@code readingUser} */
static void checkFileContentDirect(URI uri, Path name, byte[] expected,
    int readOffset, String readingUser, Configuration conf,
    boolean legacyShortCircuitFails)
    throws IOException, InterruptedException {
  // Ensure short circuit is enabled
  DistributedFileSystem fs = getFileSystem(readingUser, uri, conf);
  ClientContext clientContext = ClientContext.getFromConf(conf);
  if (legacyShortCircuitFails) {
    assertTrue(clientContext.getDisableLegacyBlockReaderLocal());
  }
  
  HdfsDataInputStream stm = (HdfsDataInputStream)fs.open(name);

  ByteBuffer actual = ByteBuffer.allocateDirect(expected.length - readOffset);

  IOUtils.skipFully(stm, readOffset);

  actual.limit(3);

  //Read a small number of bytes first.
  int nread = stm.read(actual);
  actual.limit(nread + 2);
  nread += stm.read(actual);

  // Read across chunk boundary
  actual.limit(Math.min(actual.capacity(), nread + 517));
  nread += stm.read(actual);
  checkData(arrayFromByteBuffer(actual), readOffset, expected, nread,
      "A few bytes");
  //Now read rest of it
  actual.limit(actual.capacity());
  while (actual.hasRemaining()) {
    int nbytes = stm.read(actual);

    if (nbytes < 0) {
      throw new EOFException("End of file reached before reading fully.");
    }
    nread += nbytes;
  }
  checkData(arrayFromByteBuffer(actual), readOffset, expected, "Read 3");
  if (legacyShortCircuitFails) {
    assertTrue(clientContext.getDisableLegacyBlockReaderLocal());
  }
  stm.close();
}
 

import org.apache.hadoop.hdfs.client.HdfsDataInputStream; //导入方法依赖的package包/类
/** Check the file content, reading as user {@code readingUser} */
static void checkFileContentDirect(URI uri, Path name, byte[] expected,
    int readOffset, String readingUser, Configuration conf,
    boolean legacyShortCircuitFails)
    throws IOException, InterruptedException {
  // Ensure short circuit is enabled
  DistributedFileSystem fs = getFileSystem(readingUser, uri, conf);
  ClientContext clientContext = ClientContext.getFromConf(conf);
  if (legacyShortCircuitFails) {
    assertTrue(clientContext.getDisableLegacyBlockReaderLocal());
  }

  HdfsDataInputStream stm = (HdfsDataInputStream)fs.open(name);

  ByteBuffer actual = ByteBuffer.allocateDirect(expected.length - readOffset);

  IOUtils.skipFully(stm, readOffset);

  actual.limit(3);

  //Read a small number of bytes first.
  int nread = stm.read(actual);
  actual.limit(nread + 2);
  nread += stm.read(actual);

  // Read across chunk boundary
  actual.limit(Math.min(actual.capacity(), nread + 517));
  nread += stm.read(actual);
  checkData(arrayFromByteBuffer(actual), readOffset, expected, nread,
      "A few bytes");
  //Now read rest of it
  actual.limit(actual.capacity());
  while (actual.hasRemaining()) {
    int nbytes = stm.read(actual);

    if (nbytes < 0) {
      throw new EOFException("End of file reached before reading fully.");
    }
    nread += nbytes;
  }
  checkData(arrayFromByteBuffer(actual), readOffset, expected, "Read 3");
  if (legacyShortCircuitFails) {
    assertTrue(clientContext.getDisableLegacyBlockReaderLocal());
  }
  stm.close();
}
 

import org.apache.hadoop.hdfs.client.HdfsDataInputStream; //导入方法依赖的package包/类
/** Check the file content, reading as user {@code readingUser} */
static void checkFileContentDirect(URI uri, Path name, byte[] expected,
    int readOffset, String readingUser, Configuration conf,
    boolean legacyShortCircuitFails)
    throws IOException, InterruptedException {
  // Ensure short circuit is enabled
  DistributedFileSystem fs = getFileSystem(readingUser, uri, conf);
  if (legacyShortCircuitFails) {
    assertTrue(fs.getClient().useLegacyBlockReaderLocal());
  }
  
  HdfsDataInputStream stm = (HdfsDataInputStream)fs.open(name);

  ByteBuffer actual = ByteBuffer.allocateDirect(expected.length - readOffset);

  IOUtils.skipFully(stm, readOffset);

  actual.limit(3);

  //Read a small number of bytes first.
  int nread = stm.read(actual);
  actual.limit(nread + 2);
  nread += stm.read(actual);

  // Read across chunk boundary
  actual.limit(Math.min(actual.capacity(), nread + 517));
  nread += stm.read(actual);
  checkData(arrayFromByteBuffer(actual), readOffset, expected, nread,
      "A few bytes");
  //Now read rest of it
  actual.limit(actual.capacity());
  while (actual.hasRemaining()) {
    int nbytes = stm.read(actual);

    if (nbytes < 0) {
      throw new EOFException("End of file reached before reading fully.");
    }
    nread += nbytes;
  }
  checkData(arrayFromByteBuffer(actual), readOffset, expected, "Read 3");
  if (legacyShortCircuitFails) {
    assertFalse(fs.getClient().useLegacyBlockReaderLocal());
  }
  stm.close();
}