Java FSDataInputStream.read方法代码示例

import org.apache.hadoop.fs.FSDataInputStream; //导入方法依赖的package包/类
@Test
public void testOpenFileTwice() throws Throwable {
  describe("verify that two opened file streams are independent");
  Path path = path("testopenfiletwice.txt");
  byte[] block = dataset(TEST_FILE_LEN, 0, 255);
  //this file now has a simple rule: offset => value
  createFile(getFileSystem(), path, false, block);
  //open first
  FSDataInputStream instream1 = getFileSystem().open(path);
  int c = instream1.read();
  assertEquals(0,c);
  FSDataInputStream instream2 = null;
  try {
    instream2 = getFileSystem().open(path);
    assertEquals("first read of instream 2", 0, instream2.read());
    assertEquals("second read of instream 1", 1, instream1.read());
    instream1.close();
    assertEquals("second read of instream 2", 1, instream2.read());
    //close instream1 again
    instream1.close();
  } finally {
    IOUtils.closeStream(instream1);
    IOUtils.closeStream(instream2);
  }
}
 

import org.apache.hadoop.fs.FSDataInputStream; //导入方法依赖的package包/类
/**
 * 查看输出结果
 *
 * @param path
 */
public void print(String path) {
    log.info("mapreduce输出结果:...................................................");
    DistributedFileSystem distributedFileSystem = distributedFileSystem();
    try {
        FileStatus[] fileStatuses = distributedFileSystem.listStatus(new Path(path));
        for (FileStatus fs : fileStatuses) {
            log.info(fs);
            FSDataInputStream fsDataInputStream = distributedFileSystem.open(fs.getPath());
            byte[] bs = new byte[fsDataInputStream.available()];
            fsDataInputStream.read(bs);
            log.info("\n" + new String(bs) + "\n");
        }
    } catch (IOException e) {
        log.error(e);
    } finally {
        close(distributedFileSystem);
    }
}
 

import org.apache.hadoop.fs.FSDataInputStream; //导入方法依赖的package包/类
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testRenameFile() throws Exception {
  assumeRenameSupported();

  final Path old = new Path("/test/alice/file");
  final Path newPath = new Path("/test/bob/file");
  fs.mkdirs(newPath.getParent());
  final FSDataOutputStream fsDataOutputStream = fs.create(old);
  final byte[] message = "Some data".getBytes();
  fsDataOutputStream.write(message);
  fsDataOutputStream.close();

  assertTrue(fs.exists(old));
  rename(old, newPath, true, false, true);

  final FSDataInputStream bobStream = fs.open(newPath);
  final byte[] bytes = new byte[512];
  final int read = bobStream.read(bytes);
  bobStream.close();
  final byte[] buffer = new byte[read];
  System.arraycopy(bytes, 0, buffer, 0, read);
  assertEquals(new String(message), new String(buffer));
}
 

import org.apache.hadoop.fs.FSDataInputStream; //导入方法依赖的package包/类
/**
 * Verify that two files have different contents.
 *
 * @param fs The file system containing the two files.
 * @param p1 The path of the first file.
 * @param p2 The path of the second file.
 * @param len The length of the two files.
 * @throws IOException
 */
public static void verifyFilesNotEqual(FileSystem fs, Path p1, Path p2,
    int len)
        throws IOException {
  final FSDataInputStream in1 = fs.open(p1);
  final FSDataInputStream in2 = fs.open(p2);
  try {
    for (int i = 0; i < len; i++) {
      if (in1.read() != in2.read()) {
        return;
      }
    }
    fail("files are equal, but should not be");
  } finally {
    in1.close();
    in2.close();
  }
}
 

import org.apache.hadoop.fs.FSDataInputStream; //导入方法依赖的package包/类
/**
 * 1. create files with dfs
 * 2. write 1 byte
 * 3. close file
 * 4. open the same file
 * 5. read the 1 byte and compare results
 */
static void write1byte(String methodName) throws IOException {
  final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf
      ).numDataNodes(REPLICATION + 1).build();
  final FileSystem dfs = cluster.getFileSystem();
  try {
    final Path p = new Path("/" + methodName + "/foo");
    final FSDataOutputStream out = createFile(dfs, p);
    out.write(1);
    out.close();
    
    final FSDataInputStream in = dfs.open(p);
    final int b = in.read();
    in.close();
    Assert.assertEquals(1, b);
  }
  finally {
    dfs.close();
    cluster.shutdown();
  }
}
 

import org.apache.hadoop.fs.FSDataInputStream; //导入方法依赖的package包/类
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testWriteReadFile() throws Exception {
  final Path f = new Path("/test/test");
  final FSDataOutputStream fsDataOutputStream = fs.create(f);
  final String message = "Test string";
  fsDataOutputStream.write(message.getBytes());
  fsDataOutputStream.close();
  assertExists("created file", f);
  FSDataInputStream open = null;
  try {
    open = fs.open(f);
    final byte[] bytes = new byte[512];
    final int read = open.read(bytes);
    final byte[] buffer = new byte[read];
    System.arraycopy(bytes, 0, buffer, 0, read);
    assertEquals(message, new String(buffer));
  } finally {
    fs.delete(f, false);
    IOUtils.closeStream(open);
  }
}
 

import org.apache.hadoop.fs.FSDataInputStream; //导入方法依赖的package包/类
private Pair<Integer,ArrayList<Microblog>> readPartitionMicroblogs(
        DirectoryEntry dirEntry, Rectangle queryRange) {
    String hdfsPath = getDirectoryId()+dirEntry.getFilePath();
    Integer partitionId = dirEntry.getKeyGeo();
    long offset = dirEntry.getOffset();
    int length = dirEntry.getLength();

    FileSystem hdfs = KiteInstance.hdfs();
    try {
        FSDataInputStream block = hdfs.open(new Path(hdfsPath),
                ConstantsAndDefaults.BUFFER_SIZE_BYTES);
        byte [] keyData = new byte[length];
        int read = block.read(offset,keyData,0,length);
        if(read <= 0)
            return null;

        Pair<Integer,ArrayList<Microblog>> singleHashEntry =
                Serializer.deserializeSpatialEntry(keyData,
                        stream.getScheme());

        return new Pair<>(singleHashEntry.getKey(),
                filterSpatialRange(singleHashEntry.getValue(), queryRange));
    } catch (IOException e) {
        String errMsg = "Unable to read HDFS path "+hdfsPath;
        errMsg += System.lineSeparator();
        errMsg += "Error: "+e.getMessage();
        KiteInstance.logError(errMsg);
        System.err.println(errMsg);
        return null;
    }
}
 

import org.apache.hadoop.fs.FSDataInputStream; //导入方法依赖的package包/类
/**
 * 1. create files with dfs
 * 2. write MIN_N_PACKET to MAX_N_PACKET packets
 * 3. close file
 * 4. open the same file
 * 5. read the bytes and compare results
 */
private static void writeSeveralPackets(String methodName) throws IOException {
  final Random r = FiTestUtil.RANDOM.get();
  final int nPackets = FiTestUtil.nextRandomInt(MIN_N_PACKET, MAX_N_PACKET + 1);
  final int lastPacketSize = FiTestUtil.nextRandomInt(1, PACKET_SIZE + 1);
  final int size = (nPackets - 1)*PACKET_SIZE + lastPacketSize;

  FiTestUtil.LOG.info("size=" + size + ", nPackets=" + nPackets
      + ", lastPacketSize=" + lastPacketSize);

  final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf
      ).numDataNodes(REPLICATION + 2).build();
  final FileSystem dfs = cluster.getFileSystem();
  try {
    final Path p = new Path("/" + methodName + "/foo");
    final FSDataOutputStream out = createFile(dfs, p);

    final long seed = r.nextLong();
    final Random ran = new Random(seed);
    ran.nextBytes(bytes);
    out.write(bytes, 0, size);
    out.close();

    final FSDataInputStream in = dfs.open(p);
    int totalRead = 0;
    int nRead = 0;
    while ((nRead = in.read(toRead, totalRead, size - totalRead)) > 0) {
      totalRead += nRead;
    }
    Assert.assertEquals("Cannot read file.", size, totalRead);
    for (int i = 0; i < size; i++) {
      Assert.assertTrue("File content differ.", bytes[i] == toRead[i]);
    }
  }
  finally {
    dfs.close();
    cluster.shutdown();
  }
}
 

import org.apache.hadoop.fs.FSDataInputStream; //导入方法依赖的package包/类
private static void skipBuffer(FSDataInputStream in, byte v) throws IOException {
  byte[] data = new byte[8192];
  try {
    int n;
    while ((n = in.read(data)) == data.length) {
      for (int i = 0; i < data.length; ++i) {
        if (data[i] != v)
          throw new Exception("File changed");
      }
    }
  } catch (Exception e) {
  }
}
 

import org.apache.hadoop.fs.FSDataInputStream; //导入方法依赖的package包/类
public static void truncateFile(FileSystem fs, Path src, Path dst) throws IOException {
  FileStatus fst = fs.getFileStatus(src);
  long len = fst.getLen();
  len = len / 2 ;

  // create a truncated hfile
  FSDataOutputStream fdos = fs.create(dst);
  byte[] buf = new byte[(int)len];
  FSDataInputStream fdis = fs.open(src);
  fdis.read(buf);
  fdos.write(buf);
  fdis.close();
  fdos.close();
}
 

import org.apache.hadoop.fs.FSDataInputStream; //导入方法依赖的package包/类
public static Dictionary readDictionary(FSDataInputStream in, ColumnDescriptor column, PageHeaderWithOffset pageHeader, BytesDecompressor decompressor) throws IOException {
  in.seek(pageHeader.getOffset());
  final byte[] data = new byte[pageHeader.getPageHeader().getCompressed_page_size()];
  int read = in.read(data);
  if (read != data.length) {
    throw new IOException(format("Failed to read dictionary page, read %d bytes, expected %d", read, data.length));
  }
  final DictionaryPage dictionaryPage = new DictionaryPage(
    decompressor.decompress(BytesInput.from(data), pageHeader.getPageHeader().getUncompressed_page_size()),
    pageHeader.getPageHeader().getDictionary_page_header().getNum_values(),
    CONVERTER.getEncoding(pageHeader.getPageHeader().getDictionary_page_header().getEncoding()));
  return dictionaryPage.getEncoding().initDictionary(column, dictionaryPage);
}
 

import org.apache.hadoop.fs.FSDataInputStream; //导入方法依赖的package包/类
/**
 * Load from a Hadoop filesystem
 * @param fs filesystem
 * @param path path
 * @return a loaded CD
 * @throws IOException IO problems
 * @throws EOFException if not enough bytes were read in
 * @throws JsonParseException parse problems
 * @throws JsonMappingException O/J mapping problems
 */
public T load(FileSystem fs, Path path)
    throws IOException, JsonParseException, JsonMappingException {
  FileStatus status = fs.getFileStatus(path);
  long len = status.getLen();
  byte[] b = new byte[(int) len];
  FSDataInputStream dataInputStream = fs.open(path);
  int count = dataInputStream.read(b);
  if (count != len) {
    throw new EOFException(path.toString() + ": read finished prematurely");
  }
  return fromBytes(path.toString(), b);
}
 

import org.apache.hadoop.fs.FSDataInputStream; //导入方法依赖的package包/类
/**
 * Read in-memory block with Short Circuit Read
 * Note: the test uses faked RAM_DISK from physical disk.
 */
@Test
public void testRamDiskShortCircuitRead()
  throws IOException, InterruptedException {
  startUpCluster(REPL_FACTOR,
    new StorageType[]{RAM_DISK, DEFAULT},
    2 * BLOCK_SIZE - 1, true);  // 1 replica + delta, SCR read
  final String METHOD_NAME = GenericTestUtils.getMethodName();
  final int SEED = 0xFADED;
  Path path = new Path("/" + METHOD_NAME + ".dat");

  makeRandomTestFile(path, BLOCK_SIZE, true, SEED);
  ensureFileReplicasOnStorageType(path, RAM_DISK);

  // Sleep for a short time to allow the lazy writer thread to do its job
  Thread.sleep(3 * LAZY_WRITER_INTERVAL_SEC * 1000);

  //assertThat(verifyReadRandomFile(path, BLOCK_SIZE, SEED), is(true));
  FSDataInputStream fis = fs.open(path);

  // Verify SCR read counters
  try {
    fis = fs.open(path);
    byte[] buf = new byte[BUFFER_LENGTH];
    fis.read(0, buf, 0, BUFFER_LENGTH);
    HdfsDataInputStream dfsis = (HdfsDataInputStream) fis;
    Assert.assertEquals(BUFFER_LENGTH,
      dfsis.getReadStatistics().getTotalBytesRead());
    Assert.assertEquals(BUFFER_LENGTH,
      dfsis.getReadStatistics().getTotalShortCircuitBytesRead());
  } finally {
    fis.close();
    fis = null;
  }
}
 

import org.apache.hadoop.fs.FSDataInputStream; //导入方法依赖的package包/类
private void doPread(FSDataInputStream stm, long position, byte[] buffer,
                     int offset, int length) throws IOException {
  int nread = 0;
  long totalRead = 0;
  DFSInputStream dfstm = null;

  if (stm.getWrappedStream() instanceof DFSInputStream) {
    dfstm = (DFSInputStream) (stm.getWrappedStream());
    totalRead = dfstm.getReadStatistics().getTotalBytesRead();
  }

  while (nread < length) {
    int nbytes =
        stm.read(position + nread, buffer, offset + nread, length - nread);
    assertTrue("Error in pread", nbytes > 0);
    nread += nbytes;
  }

  if (dfstm != null) {
    if (isHedgedRead) {
      assertTrue("Expected read statistic to be incremented", length <= dfstm
          .getReadStatistics().getTotalBytesRead() - totalRead);
    } else {
      assertEquals("Expected read statistic to be incremented", length, dfstm
          .getReadStatistics().getTotalBytesRead() - totalRead);
    }
  }
}
 

import org.apache.hadoop.fs.FSDataInputStream; //导入方法依赖的package包/类
/** Test whether corrupt replicas are detected correctly during pipeline
 * recoveries.
 */
@Test
public void testPipelineRecoveryForLastBlock() throws IOException {
  DFSClientFaultInjector faultInjector
      = Mockito.mock(DFSClientFaultInjector.class);
  DFSClientFaultInjector oldInjector = DFSClientFaultInjector.instance;
  DFSClientFaultInjector.instance = faultInjector;
  Configuration conf = new HdfsConfiguration();

  conf.setInt(DFSConfigKeys.DFS_CLIENT_BLOCK_WRITE_LOCATEFOLLOWINGBLOCK_RETRIES_KEY, 3);
  MiniDFSCluster cluster = null;

  try {
    int numDataNodes = 3;
    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDataNodes).build();
    cluster.waitActive();
    FileSystem fileSys = cluster.getFileSystem();

    Path file = new Path("dataprotocol1.dat");
    Mockito.when(faultInjector.failPacket()).thenReturn(true);
    DFSTestUtil.createFile(fileSys, file, 68000000L, (short)numDataNodes, 0L);

    // At this point, NN should have accepted only valid replicas.
    // Read should succeed.
    FSDataInputStream in = fileSys.open(file);
    try {
      int c = in.read();
      // Test will fail with BlockMissingException if NN does not update the
      // replica state based on the latest report.
    } catch (org.apache.hadoop.hdfs.BlockMissingException bme) {
      Assert.fail("Block is missing because the file was closed with"
          + " corrupt replicas.");
    }
  } finally {
    DFSClientFaultInjector.instance = oldInjector;
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}