Java DistributedRaidFileSystem类代码示例

import org.apache.hadoop.hdfs.DistributedRaidFileSystem; //导入依赖的package包/类
/**
 * Helper function of Encoder.
 */
public List<List<Block>> getSrcStripes() {
  List<List<Block>> srcStripes = new ArrayList<List<Block>>();
  for (int i=0;i<srcStripeList.size();i++) {
    List<BlockInfo> biList=srcStripeList.get(i);
    List curSrcStripe = new ArrayList<Block>();
    for (int j=0;j<biList.size();j++) {
      int fileIdx = biList.get(j).fileIdx;
      int blockId = biList.get(j).blockId;
      FileStatus curFs = lfs.get(fileIdx);

      try {
        if (fs instanceof DistributedRaidFileSystem) {
          curSrcStripe.add(
              ((DistributedRaidFileSystem)fs).toDistributedFileSystem().getLocatedBlocks(curFs.getPath(),
              0L, curFs.getLen()).get(blockId).getBlock());
        }
      } catch (IOException e) {
        ;
      }
    }
    srcStripes.add(curSrcStripe);
  }
  return srcStripes;
}
 

import org.apache.hadoop.hdfs.DistributedRaidFileSystem; //导入依赖的package包/类
public GenThread(Configuration conf, Path input, Path output,
    RunTimeConstants rtc) throws IOException {
  this.inputPath = input;
  this.outputPath = output;
  this.fs = FileSystem.newInstance(conf);
  if (fs instanceof DistributedRaidFileSystem) {
    fs = ((DistributedRaidFileSystem)fs).getFileSystem();
  }
  this.buffer = new byte[rtc.buffer_size];
  if (test_buffer_size > rtc.buffer_size) 
    test_buffer_size = rtc.buffer_size;
}
 

import org.apache.hadoop.hdfs.DistributedRaidFileSystem; //导入依赖的package包/类
public double getEffectiveReplication() {
  if (lastRaidStatistics == null) {
    return -1;
  }
  DFSClient dfs;
  double totalPhysical;
  try {
      /* Add by RH start */
      if(FileSystem.get(conf) instanceof DistributedRaidFileSystem){
          dfs = ((DistributedRaidFileSystem)FileSystem.get(conf)).getClient();
      }else{
          dfs = ((DistributedFileSystem)FileSystem.get(conf)).getClient();
      }
      /* Add by RH end */
      /* Commented by RH start */
    //dfs = ((DistributedFileSystem)FileSystem.get(conf)).getClient();
      /* Commented by RH end */
    totalPhysical = dfs.getNSDiskStatus().getDfsUsed();
  } catch (IOException e) {
    return -1;
  }
  double notRaidedPhysical = totalPhysical;
  double totalLogical = 0;
  for (Codec codec : Codec.getCodecs()) {
    String code = codec.id;
    Statistics st = lastRaidStatistics.get(code);
    totalLogical += st.getSourceCounters(RaidState.RAIDED).getNumLogical();
    notRaidedPhysical -= st.getSourceCounters(RaidState.RAIDED).getNumBytes();
    notRaidedPhysical -= st.getParityCounters().getNumBytes();
  }
  totalLogical += notRaidedPhysical / dfs.getDefaultReplication();
  if (totalLogical == 0) {
    // divided by 0
    return -1;
  }
  return totalPhysical / totalLogical;
}
 

import org.apache.hadoop.hdfs.DistributedRaidFileSystem; //导入依赖的package包/类
public void purgeParity(String cmd, String[] args, int startIndex)
    throws IOException {
  if (startIndex + 1 >= args.length) {
    printUsage(cmd);
    throw new IllegalArgumentException("Insufficient arguments");
  }
  Path parityPath = new Path(args[startIndex]);
  AtomicLong entriesProcessed = new AtomicLong(0);
  System.err.println("Starting recursive purge of " + parityPath);

  Codec codec = Codec.getCodec(args[startIndex + 1]);
  FileSystem srcFs = parityPath.getFileSystem(conf);
  if (srcFs instanceof DistributedRaidFileSystem) {
    srcFs = ((DistributedRaidFileSystem)srcFs).getFileSystem();
  }
  FileSystem parityFs = srcFs;
  String parityPrefix = codec.parityDirectory;
  DirectoryTraversal obsoleteParityFileRetriever =
    new DirectoryTraversal(
      "Purge File ",
      java.util.Collections.singletonList(parityPath),
      parityFs,
      new PurgeMonitor.PurgeParityFileFilter(conf, codec, null,
          srcFs, parityFs,
        parityPrefix, null, entriesProcessed),
      1,
      false);
  FileStatus obsolete = null;
  while ((obsolete = obsoleteParityFileRetriever.next()) !=
            DirectoryTraversal.FINISH_TOKEN) {
    PurgeMonitor.performDelete(parityFs, obsolete.getPath(), false);
  }
}
 

import org.apache.hadoop.hdfs.DistributedRaidFileSystem; //导入依赖的package包/类
/**
 * gets the parity blocks corresponding to file
 * returns the parity blocks in case of DFS
 * and the part blocks containing parity blocks
 * in case of HAR FS
 */
private static BlockLocation[] getParityBlocks(final Path filePath,
                                        final long blockSize,
                                        final long numStripes,
                                        final RaidInfo raidInfo) 
  throws IOException {
  FileSystem parityFS = raidInfo.parityPair.getFileSystem();
  
  // get parity file metadata
  FileStatus parityFileStatus = raidInfo.parityPair.getFileStatus(); 
  long parityFileLength = parityFileStatus.getLen();

  if (parityFileLength != numStripes * raidInfo.parityBlocksPerStripe *
      blockSize) {
    throw new IOException("expected parity file of length" + 
                          (numStripes * raidInfo.parityBlocksPerStripe *
                           blockSize) +
                          " but got parity file of length " + 
                          parityFileLength);
  }

  BlockLocation[] parityBlocks = 
    parityFS.getFileBlockLocations(parityFileStatus, 0L, parityFileLength);
  
  if (parityFS instanceof DistributedFileSystem ||
      parityFS instanceof DistributedRaidFileSystem) {
    long parityBlockSize = parityFileStatus.getBlockSize();
    if (parityBlockSize != blockSize) {
      throw new IOException("file block size is " + blockSize + 
                            " but parity file block size is " + 
                            parityBlockSize);
    }
  } else if (parityFS instanceof HarFileSystem) {
    LOG.debug("HAR FS found");
  } else {
    LOG.warn("parity file system is not of a supported type");
  }
  
  return parityBlocks;
}
 

import org.apache.hadoop.hdfs.DistributedRaidFileSystem; //导入依赖的package包/类
private DistributedRaidFileSystem getRaidFS() throws IOException {
  DistributedFileSystem dfs = (DistributedFileSystem)fileSys;
  Configuration clientConf = new Configuration(conf);
  clientConf.set("fs.hdfs.impl", 
      "org.apache.hadoop.hdfs.DistributedRaidFileSystem");
  clientConf.set("fs.raid.underlyingfs.impl", 
      "org.apache.hadoop.hdfs.DistributedFileSystem");
  clientConf.setBoolean("fs.hdfs.impl.disable.cache", true);
  URI dfsUri = dfs.getUri();
  return (DistributedRaidFileSystem)FileSystem.get(dfsUri, clientConf);
}
 

import org.apache.hadoop.hdfs.DistributedRaidFileSystem; //导入依赖的package包/类
private DistributedRaidFileSystem getRaidFS() throws IOException {
  DistributedFileSystem dfs = (DistributedFileSystem)fileSys;
  Configuration clientConf = new Configuration(conf);
  clientConf.set("fs.hdfs.impl", 
           "org.apache.hadoop.hdfs.DistributedRaidFileSystem");
  clientConf.set("fs.raid.underlyingfs.impl", 
           "org.apache.hadoop.hdfs.DistributedFileSystem");
  clientConf.setBoolean("fs.hdfs.impl.disable.cache", true);
  URI dfsUri = dfs.getUri();
  return (DistributedRaidFileSystem)FileSystem.get(dfsUri, clientConf);
}
 

import org.apache.hadoop.hdfs.DistributedRaidFileSystem; //导入依赖的package包/类
public void testRenameHar() throws Exception {
  try {
    mySetup("xor", 1);
    
    Path[] testPathList = new Path[] {
        new Path ("/user/dikang/raidtest/rename/f1"),
        new Path ("/user/dikang/raidtest/rename/f2"),
        new Path ("/user/dikang/raidtest/rename/f3")};
    
    Path destHarPath = new Path ("/destraid/user/dikang/raidtest/rename");
    
    DistributedRaidFileSystem raidFs = getRaidFS();
    for (Path srcPath : testPathList) {
      TestRaidDfs.createTestFilePartialLastBlock(fileSys, srcPath, 
          1, 8, 8192L);
    }
    
    raidFs.mkdirs(destHarPath);
    raidFs.mkdirs(new Path(destHarPath, "rename" + RaidNode.HAR_SUFFIX));
    
    raidFs.rename(new Path("/user/dikang/raidtest"), 
        new Path("/user/dikang/raidtest1"));
    fail("Expected fail for HAR rename");
  } catch (IOException ie) {
    String message = ie.getMessage();
    assertTrue(message.contains("HAR dir"));
  } finally {
    stopCluster();
  }
}
 

import org.apache.hadoop.hdfs.DistributedRaidFileSystem; //导入依赖的package包/类
public void testRename() throws Exception {
  try {
    
    long[] crcs = new long[3];
    int[] seeds = new int[3];
    short repl = 1;
    Path dirPath = new Path("/user/dikang/raidtest");
    
    mySetup();
    DistributedRaidFileSystem raidFs = getRaidFS();
    Path[] files = TestRaidDfs.createTestFiles(dirPath,
        fileSizes, blockSizes, crcs, seeds, fileSys, (short)1);
    FileStatus stat = raidFs.getFileStatus(dirPath); 
    Codec codec = Codec.getCodec("dir-rs");
    RaidNode.doRaid(conf, stat, new Path(codec.parityDirectory), codec,
        new RaidNode.Statistics(), RaidUtils.NULL_PROGRESSABLE,
      false, repl, repl);
    
    Path destPath = new Path("/user/dikang/raidtest_new");
    
    assertTrue(raidFs.exists(dirPath));
    assertFalse(raidFs.exists(destPath));
    
    ParityFilePair parity = ParityFilePair.getParityFile(
        codec, stat, conf);
    Path srcParityPath = parity.getPath();
    assertTrue(raidFs.exists(srcParityPath));
    // do the rename file
    assertTrue(raidFs.rename(dirPath, destPath));
    // verify the results.
    assertFalse(raidFs.exists(dirPath));
    assertTrue(raidFs.exists(destPath));
    assertFalse(raidFs.exists(srcParityPath));
    FileStatus srcDest = raidFs.getFileStatus(destPath);
    parity = ParityFilePair.getParityFile(codec, srcDest, conf);
    assertTrue(raidFs.exists(parity.getPath()));
  } finally {
    stopCluster();
  }
}
 

import org.apache.hadoop.hdfs.DistributedRaidFileSystem; //导入依赖的package包/类
public void testDeleteOneFile() throws Exception {
  try {
    long[] crcs = new long[3];
    int[] seeds = new int[3];
    short repl = 1;
    Path dirPath = new Path("/user/dikang/raidtest");
    
    mySetup();
    DistributedRaidFileSystem raidFs = getRaidFS();
    Path[] files = TestRaidDfs.createTestFiles(dirPath,
        fileSizes, blockSizes, crcs, seeds, fileSys, (short)1);
    FileStatus stat = raidFs.getFileStatus(dirPath); 
    Codec codec = Codec.getCodec("dir-rs");
    RaidNode.doRaid(conf, stat, new Path(codec.parityDirectory), codec,
        new RaidNode.Statistics(), RaidUtils.NULL_PROGRESSABLE,
      false, repl, repl);
    
    ParityFilePair parity = ParityFilePair.getParityFile(
        codec, stat, conf);
    Path srcParityPath = parity.getPath();
    assertTrue(raidFs.exists(srcParityPath));
    
    // delete one file
    assertTrue(raidFs.delete(files[0]));
    // verify the results
    assertFalse(raidFs.exists(files[0]));
    // we still have the parity file
    assertTrue(raidFs.exists(srcParityPath));
    
    // delete the left files
    assertTrue(raidFs.delete(files[1]));
    assertTrue(raidFs.delete(files[2]));
    
    // we will not touch the parity file.
    assertTrue(raidFs.exists(srcParityPath));
  } finally {
    stopCluster();
  }
}
 

import org.apache.hadoop.hdfs.DistributedRaidFileSystem; //导入依赖的package包/类
public void testDeleteDirRaidedFile() throws Exception {
  
  long[] crcs = new long[3];
  int[] seeds = new int[3];
  short repl = 1;
  Path dirPath = new Path("/user/dikang/raidtest");
  
  mySetup();
  //disable trash
  conf.setInt("fs.trash.interval", 0);
  
  DistributedRaidFileSystem raidFs = getRaidFS();
  Path[] files = TestRaidDfs.createTestFiles(dirPath,
      fileSizes, blockSizes, crcs, seeds, fileSys, (short)1);
  FileStatus stat = raidFs.getFileStatus(dirPath); 
  Codec codec = Codec.getCodec("dir-rs");
  RaidNode.doRaid(conf, stat, new Path(codec.parityDirectory), codec,
      new RaidNode.Statistics(), RaidUtils.NULL_PROGRESSABLE,
    false, repl, repl);
  
  try {
    raidFs.delete(files[0]);
    fail();
  } catch (Exception ex) {
    LOG.warn("Excepted error: " + ex.getMessage(), ex);
  } finally {
    stopCluster();
  }
}
 

import org.apache.hadoop.hdfs.DistributedRaidFileSystem; //导入依赖的package包/类
static private DistributedRaidFileSystem getRaidFS(FileSystem fileSys,
    Configuration conf)
    throws IOException {
  DistributedFileSystem dfs = (DistributedFileSystem)fileSys;
  Configuration clientConf = new Configuration(conf);
  clientConf.set("fs.hdfs.impl", "org.apache.hadoop.hdfs.DistributedRaidFileSystem");
  clientConf.set("fs.raid.underlyingfs.impl", "org.apache.hadoop.hdfs.DistributedFileSystem");
  clientConf.setBoolean("fs.hdfs.impl.disable.cache", true);
  URI dfsUri = dfs.getUri();
  return (DistributedRaidFileSystem)FileSystem.get(dfsUri, clientConf);
}
 

import org.apache.hadoop.hdfs.DistributedRaidFileSystem; //导入依赖的package包/类
public void purgeParity(String cmd, String[] args, int startIndex)
    throws IOException {
  if (startIndex + 1 >= args.length) {
    printUsage(cmd);
    throw new IllegalArgumentException("Insufficient arguments");
  }
  Path parityPath = new Path(args[startIndex]);
  AtomicLong entriesProcessed = new AtomicLong(0);
  System.err.println("Starting recursive purge of " + parityPath);

  Codec codec = Codec.getCodec(args[startIndex + 1]);
  FileSystem srcFs = parityPath.getFileSystem(conf);
  if (srcFs instanceof DistributedRaidFileSystem) {
    srcFs = ((DistributedRaidFileSystem)srcFs).getFileSystem();
  }
  FileSystem parityFs = srcFs;
  String parityPrefix = codec.parityDirectory;
  DirectoryTraversal obsoleteParityFileRetriever =
    new DirectoryTraversal(
      "Purge File ",
      java.util.Collections.singletonList(parityPath),
      parityFs,
      new PurgeMonitor.PurgeParityFileFilter(conf, codec, srcFs, parityFs,
        parityPrefix, null, entriesProcessed),
      1,
      false);
  FileStatus obsolete = null;
  while ((obsolete = obsoleteParityFileRetriever.next()) !=
            DirectoryTraversal.FINISH_TOKEN) {
    PurgeMonitor.performDelete(parityFs, obsolete.getPath(), false);
  }
}
 

import org.apache.hadoop.hdfs.DistributedRaidFileSystem; //导入依赖的package包/类
private static DistributedFileSystem getDFS(FileSystem fs)
  throws IOException {
  if (fs instanceof DistributedRaidFileSystem)
    fs = ((DistributedRaidFileSystem)fs).getFileSystem();
  return (DistributedFileSystem)fs;
}
 

import org.apache.hadoop.hdfs.DistributedRaidFileSystem; //导入依赖的package包/类
public void checkFile(String cmd, String[] args, int startIndex)
    throws IOException {
  if (startIndex >= args.length) {
    printUsage(cmd);
    throw new IllegalArgumentException("Insufficient arguments");
  }
  for (int i = startIndex; i < args.length; i++) {
    Path p = new Path(args[i]);
    FileSystem fs = p.getFileSystem(conf);
    // if we got a raid fs, get the underlying fs 
    if (fs instanceof DistributedRaidFileSystem) {
      fs = ((DistributedRaidFileSystem) fs).getFileSystem();
    }
    // We should be able to cast at this point.
    DistributedFileSystem dfs = (DistributedFileSystem) fs;
    RemoteIterator<Path> corruptIt = dfs.listCorruptFileBlocks(p);
    int count = 0;
    while (corruptIt.hasNext()) {
      count++;
      Path corruptFile = corruptIt.next();
      // Result of checking.
      String result = null;
      FileStatus stat = fs.getFileStatus(corruptFile);
      if (stat.getReplication() < fs.getDefaultReplication()) {
        RaidInfo raidInfo = RaidUtils.getFileRaidInfo(stat, conf);
        if (raidInfo.codec == null) {
          result = "Below default replication but no parity file found";
        } else {
          boolean notRecoverable = isFileCorrupt(dfs, stat);
          if (notRecoverable) {
            result = "Missing too many blocks to be recovered " + 
              "using parity file " + raidInfo.parityPair.getPath();
          } else {
            result = "Has missing blocks but can be read using parity file " +
              raidInfo.parityPair.getPath();
          }
        }
      } else {
        result = "At default replication, not raided";
      }
      out.println("Result of checking " + corruptFile + " : " +
        result);
    }
    out.println("Found " + count + " files with missing blocks");
  }
}