Java DFSInputStream.close方法代码示例

import org.apache.hadoop.hdfs.DFSInputStream; //导入方法依赖的package包/类
/**
 * lists, and then reads the first byte in all of the files in
 * this hostname/process' namespace
 * @return -1 on error, 0 on success
 */
private static int readDFSPaths() {
	if (listDFSPaths() != 0) {
		return -1;
	}
	
	try{ 
 	for (Map.Entry<String, OutputStream> file : files_.entrySet()) {
 		
 		long startTime = System.nanoTime();
 		DFSInputStream os = dfsClient_.open(file.getKey());
      timingOpen_.add(new Double((System.nanoTime() - startTime)/(1E9)));
      
 		os.read();
 		os.close();
 	}
	} catch (IOException e) {
		e.printStackTrace();
	}
	
	return 0;
}
 

import org.apache.hadoop.hdfs.DFSInputStream; //导入方法依赖的package包/类
private void addDirToMaps(Path dir, DFSClient client) throws IOException {
  FileStatus[] children = dfs.listStatus(dir);

  if (children == null) return;

  for (FileStatus child: children) {
    if (!child.isDir()) { // get block ids for file
      Path path = child.getPath(); // paths will be unique
      fileMap.put(path, new ArrayList<Long>());

      DFSInputStream stm = client.open(child.getPath().toUri().getPath());
      LocatedBlocks blocks = stm.fetchLocatedBlocks();
      stm.close();

      for (int i = 0; i < blocks.locatedBlockCount(); i++) {
        Long blockId = blocks.get(i).getBlock().getBlockId();
        fileMap.get(path).add(blockId); // add to file block list
        blockRefMap.put(blockId, null); // mark as unrefereced
      }
    }
    else {
      // If child is a directory, recurse on it
      addDirToMaps(child.getPath(), client);
    }
  }
}
 

import org.apache.hadoop.hdfs.DFSInputStream; //导入方法依赖的package包/类
private byte[] cacheInitialContents() throws IOException {
  HdfsFileStatus status = dfsClient.getFileInfo(name);
  byte[] content = new byte[(int)status.getLen()];
  DFSInputStream in = null;
  try {
    in = dfsClient.open(name);
    IOUtils.readFully(in, content, 0, content.length);
  } finally {
    in.close();
  }
  return content;
}
 

import org.apache.hadoop.hdfs.DFSInputStream; //导入方法依赖的package包/类
public void checkSalvagedRemains() throws IOException {
  int chainIdx = 0;
  HdfsFileStatus status = dfsClient.getFileInfo(name);
  long length = status.getLen();
  int numBlocks = (int)((length + blockSize - 1) / blockSize);
  DFSInputStream in = null;
  byte[] blockBuffer = new byte[blockSize];

  try {
    for (int blockIdx = 0; blockIdx < numBlocks; blockIdx++) {
      if (blocksToCorrupt.contains(blockIdx)) {
        if (in != null) {
          in.close();
          in = null;
        }
        continue;
      }
      if (in == null) {
        in = dfsClient.open("/lost+found" + name + "/" + chainIdx);
        chainIdx++;
      }
      int len = blockBuffer.length;
      if (blockIdx == (numBlocks - 1)) {
        // The last block might not be full-length
        len = (int)(in.getFileLength() % blockSize);
        if (len == 0) len = blockBuffer.length;
      }
      IOUtils.readFully(in, blockBuffer, 0, len);
      int startIdx = blockIdx * blockSize;
      for (int i = 0; i < len; i++) {
        if (initialContents[startIdx + i] != blockBuffer[i]) {
          throw new IOException("salvaged file " + name + " differed " +
          "from what we expected on block " + blockIdx);
        }
      }
    }
  } finally {
    IOUtils.cleanup(null, in);
  }
}
 

import org.apache.hadoop.hdfs.DFSInputStream; //导入方法依赖的package包/类
public void checkSalvagedRemains() throws IOException {
  int chainIdx = 0;
  HdfsFileStatus status = dfsClient.getFileInfo(name);
  long length = status.getLen();
  int numBlocks = (int)((length + blockSize - 1) / blockSize);
  DFSInputStream in = null;
  byte[] blockBuffer = new byte[blockSize];

  try {
    for (int blockIdx = 0; blockIdx < numBlocks; blockIdx++) {
      if (blocksToCorrupt.contains(blockIdx)) {
        if (in != null) {
          in.close();
          in = null;
        }
        continue;
      }
      if (in == null) {
        in = dfsClient.open("/lost+found" + name + "/" + chainIdx);
        chainIdx++;
      }
      int len = blockBuffer.length;
      if (blockIdx == (numBlocks - 1)) {
        // The last block might not be full-length
        len = (int)(in.getFileLength() % blockSize);
        if (len == 0) len = blockBuffer.length;
      }
      IOUtils.readFully(in, blockBuffer, 0, (int)len);
      int startIdx = blockIdx * blockSize;
      for (int i = 0; i < len; i++) {
        if (initialContents[startIdx + i] != blockBuffer[i]) {
          throw new IOException("salvaged file " + name + " differed " +
          "from what we expected on block " + blockIdx);
        }
      }
    }
  } finally {
    IOUtils.cleanup(null, in);
  }
}
 

import org.apache.hadoop.hdfs.DFSInputStream; //导入方法依赖的package包/类
private byte[] cacheInitialContents() throws IOException {
  HdfsFileStatus status = dfsClient.getFileInfo(name);
  byte[] content = new byte[(int) status.getLen()];
  DFSInputStream in = null;
  try {
    in = dfsClient.open(name);
    IOUtils.readFully(in, content, 0, content.length);
  } finally {
    in.close();
  }
  return content;
}
 

import org.apache.hadoop.hdfs.DFSInputStream; //导入方法依赖的package包/类
/**
 * Test when name-node's finger-print changes, client re-fetch the
 * name-node proxy.
 */
public void testClientUpdateMethodList() throws IOException {
  InetSocketAddress addr = cluster.getNameNode().getNameNodeDNAddress();
  DFSClient client = new DFSClient(addr, cluster.getNameNode().getConf());
  ClientProtocol oldNamenode = client.namenode;
  
  // Client's name-node proxy should keep the same if the same namenode
  // sends the same fingerprint
  //
  OutputStream os = client.create("/testClientUpdateMethodList.txt", true);
  os.write(66);
  os.close();
  TestCase.assertSame(oldNamenode, client.namenode);    
  int oldFingerprint = cluster.getNameNode().getClientProtocolMethodsFingerprint();
  TestCase.assertEquals(oldFingerprint, client.namenodeProtocolProxy
      .getMethodsFingerprint());
  
  // Namenode's fingerprint will be different to client. Client is suppsoed
  // to get a new proxy.
  //
  cluster.getNameNode().setClientProtocolMethodsFingerprint(666);
  os = client.create("/testClientUpdateMethodList1.txt", true);
  os.write(88);
  os.close();
  TestCase.assertNotSame(oldNamenode, client.namenode);
  // Since we didn't change method list of name-node, the fingerprint
  // got from the new proxy should be the same as the previous one.
  TestCase.assertEquals(oldFingerprint, client.namenodeProtocolProxy
      .getMethodsFingerprint());
  
  // Client's name-node proxy should keep the same if the same namenode
  // sends the same fingerprint
  //
  ClientProtocol namenode1 = client.namenode;
  cluster.getNameNode().setClientProtocolMethodsFingerprint(oldFingerprint);
  DFSInputStream dis = client.open("/testClientUpdateMethodList.txt");
  int val = dis.read();
  TestCase.assertEquals(66, val);
  dis.close();
  TestCase.assertSame(namenode1, client.namenode);

  // Namenode's fingerprint will be different to client. Client is suppsoed
  // to get a new proxy.
  //
  cluster.getNameNode().setClientProtocolMethodsFingerprint(888);
  dis = client.open("/testClientUpdateMethodList1.txt");
  val = dis.read();
  TestCase.assertEquals(88, val);
  dis.close();
  // Since we didn't change method list of name-node, the fingerprint
  // got from the new proxy should be the same as the previous one.
  TestCase.assertNotSame(namenode1, client.namenode);
}
 

import org.apache.hadoop.hdfs.DFSInputStream; //导入方法依赖的package包/类
public void checkSalvagedRemains() throws IOException {
  int chainIdx = 0;
  HdfsFileStatus status = dfsClient.getFileInfo(name);
  long length = status.getLen();
  int numBlocks = (int) ((length + blockSize - 1) / blockSize);
  DFSInputStream in = null;
  byte[] blockBuffer = new byte[blockSize];

  try {
    for (int blockIdx = 0; blockIdx < numBlocks; blockIdx++) {
      if (blocksToCorrupt.contains(blockIdx)) {
        if (in != null) {
          in.close();
          in = null;
        }
        continue;
      }
      if (in == null) {
        in = dfsClient.open("/lost+found" + name + "/" + chainIdx);
        chainIdx++;
      }
      int len = blockBuffer.length;
      if (blockIdx == (numBlocks - 1)) {
        // The last block might not be full-length
        len = (int) (in.getFileLength() % blockSize);
        if (len == 0) {
          len = blockBuffer.length;
        }
      }
      IOUtils.readFully(in, blockBuffer, 0, (int) len);
      int startIdx = blockIdx * blockSize;
      for (int i = 0; i < len; i++) {
        if (initialContents[startIdx + i] != blockBuffer[i]) {
          throw new IOException("salvaged file " + name + " differed " +
              "from what we expected on block " + blockIdx);
        }
      }
    }
  } finally {
    IOUtils.cleanup(null, in);
  }
}