Java LocatedFileStatus.getBlockLocations方法代码示例

import org.apache.hadoop.fs.LocatedFileStatus; //导入方法依赖的package包/类
private void verifyLocatedFileStatus(
    JobConf conf, List<LocatedFileStatus> stats)
    throws IOException {
  if (!conf.getBoolean("mapred.fileinputformat.verifysplits", true)) {
    return;
  }
  for (LocatedFileStatus stat: stats) {
    long fileLen = stat.getLen();
    long blockLenTotal = 0;
    for (BlockLocation loc: stat.getBlockLocations()) {
      blockLenTotal += loc.getLength();
    }
    if (blockLenTotal != fileLen) {
      throw new IOException("Error while getting located status, " +
        stat.getPath() + " has length " + fileLen + " but blocks total is " +
        blockLenTotal);
    }
  }
}
 

import org.apache.hadoop.fs.LocatedFileStatus; //导入方法依赖的package包/类
public int getNumberOfFiles() throws IOException {
  DistributedFileSystem dfs = (DistributedFileSystem)fs;
  RemoteIterator<LocatedFileStatus> iter = dfs.listLocatedStatus(outputPath);
  int fn = 0;
  while (iter.hasNext()) {
    LocatedFileStatus lfs = iter.next();
    if (lfs.isDir()) 
      continue;
    if (lfs.getBlockLocations().length != 1) 
      continue;
    String curHost = rtc.cur_datanode;
    for (String host: lfs.getBlockLocations()[0].getHosts()) {
      if (curHost.equals(host)){
        fn++;
        break;
      }
    }
  }
  LOG.info(" Found " + fn + " files in " + dfs.getUri());
  return fn;
}
 

import org.apache.hadoop.fs.LocatedFileStatus; //导入方法依赖的package包/类
public static void main(String[] args) throws Exception {
    String uri = "hdfs://hadoop-master:9000/";

    Configuration config = new Configuration();
    FileSystem fs = FileSystem.get(URI.create(uri), config, "root");

    FileStatus[] listStatus = fs.listStatus(new Path("/")); for (FileStatus file : listStatus) {
        System.out.println("[" + (file.isFile() ? "file" : "dir") + "] " + file.getPath().getName());
    }

    RemoteIterator<LocatedFileStatus> listFiles = fs.listFiles(new Path("/"), true);
    while (listFiles.hasNext()) {

        LocatedFileStatus fileStatus = listFiles.next();

        log.info("block size:{}",fileStatus.getBlockSize());
        log.info("owner : {}", fileStatus.getOwner());
        log.info("replication : {}" ,fileStatus.getReplication());
        log.info("permission : {}", fileStatus.getPermission());
        log.info("path name : {}",fileStatus.getPath().getName());

        log.info("========block info=========");

        BlockLocation[] blockLocations = fileStatus.getBlockLocations();

        for (BlockLocation blockLocation : blockLocations){

            log.info("block offset : {}",blockLocation.getOffset());
            log.info("block length : {}",blockLocation.getLength());

            String[] dataNodes = blockLocation.getHosts();
            for (String dataNode : dataNodes){
                log.info("dataNode :{}",dataNode);
            }
        }
    }
}
 

import org.apache.hadoop.fs.LocatedFileStatus; //导入方法依赖的package包/类
public void refreshLocalities() {
    try {
        // HashMap taks muti-thread risk here. Change to ConcurrentHashMap if it happens.
        Map<String, List<String>> newHostMap = new HashMap<>(segmentFdMap.size());

        RemoteIterator<LocatedFileStatus> files = fileSystem.listFiles(segmentRootPath, true);
        while (files.hasNext()) {
            LocatedFileStatus fileStatus = files.next();
            if (fileStatus.getLen() == 0) {
                continue;
            }
            String name = getSegmentName(fileStatus);
            if (name == null) {
                continue;
            }
            BlockLocation[] locations = fileStatus.getBlockLocations();
            if (locations.length != 1) {
                logger.error("A segment should only consisted by one block, now {}. Ignored: {}", locations.length, name);
                continue;
            }
            List<String> hosts = Arrays.asList(locations[0].getHosts());
            newHostMap.put(name, hosts);
        }

        hostMap = newHostMap;
    } catch (IOException e) {
        if (e instanceof ClosedByInterruptException) {
            logger.warn("Refresh [{}] segment locality failed by ClosedByInterruptException.", tableName);
            // Normally close interrupt.
            return;
        }
        String msg = e.getMessage();
        if (msg != null && Strings.equals(msg.trim(), "Filesystem closed")) {
            logger.warn("Refresh [{}] segment locality failed by Filesystem closed.", tableName);
            // Normally close interrupt.
            return;
        }
        logger.warn("Refresh [{}] segment locality failed.", tableName, e);
    }
}
 

import org.apache.hadoop.fs.LocatedFileStatus; //导入方法依赖的package包/类
/** 
 * Generate the list of files and make them into FileSplits.
 */ 
public List<InputSplit> getSplits(JobContext job
                                  ) throws IOException {
  long minSize = Math.max(getFormatMinSplitSize(), getMinSplitSize(job));
  long maxSize = getMaxSplitSize(job);

  // generate splits
  List<InputSplit> splits = new ArrayList<InputSplit>();
  for (LocatedFileStatus file: listLocatedStatus(job)) {
    Path path = file.getPath();
    long length = file.getLen();
    BlockLocation[] blkLocations = file.getBlockLocations();

    if ((length != 0) && isSplitable(job, path)) { 
      long blockSize = file.getBlockSize();
      long splitSize = computeSplitSize(blockSize, minSize, maxSize);

      long bytesRemaining = length;
      while (((double) bytesRemaining)/splitSize > SPLIT_SLOP) {
        int blkIndex = getBlockIndex(blkLocations, length-bytesRemaining);
        splits.add(new FileSplit(path, length-bytesRemaining, splitSize, 
                                 blkLocations[blkIndex].getHosts()));
        bytesRemaining -= splitSize;
      }
      
      if (bytesRemaining != 0) {
        splits.add(new FileSplit(path, length-bytesRemaining, bytesRemaining, 
                   blkLocations[blkLocations.length-1].getHosts()));
      }
    } else if (length != 0) {
      splits.add(new FileSplit(path, 0, length, blkLocations[0].getHosts()));
    } else { 
      //Create empty hosts array for zero length files
      splits.add(new FileSplit(path, 0, length, new String[0]));
    }
  }
  LOG.debug("Total # of splits: " + splits.size());
  return splits;
}
 

import org.apache.hadoop.fs.LocatedFileStatus; //导入方法依赖的package包/类
List<BlockInfo> getDirBlockInfos(FileSystem fs, Path dirPath)
    throws IOException {
  List<LocatedFileStatus> lfs = RaidNode.listDirectoryRaidLocatedFileStatus(conf,
      fs, dirPath);
  List<BlockInfo> result = new ArrayList<BlockInfo>();
  for (LocatedFileStatus stat: lfs) {
    for (BlockLocation loc : stat.getBlockLocations()) {
      result.add(new BlockInfo(loc, stat));
    }
  }
  return result;
}
 

import org.apache.hadoop.fs.LocatedFileStatus; //导入方法依赖的package包/类
List<BlockInfo> getBlockInfos(
  FileSystem fs, Path path, long start, long length)
    throws IOException {
  LocatedFileStatus stat = getLocatedFileStatus(fs, path);
  List<BlockInfo> result = new ArrayList<BlockInfo>();
  long end = start + length;
  if (stat != null) {
    for (BlockLocation loc : stat.getBlockLocations()) {
      if (loc.getOffset() >= start && loc.getOffset() < end) {
        result.add(new BlockInfo(loc, stat));
      }
    }
  }
  return result;
}
 

import org.apache.hadoop.fs.LocatedFileStatus; //导入方法依赖的package包/类
@Test(timeout=120000)
public void testLocatedFileStatusStorageIdsTypes() throws Exception {
  final Configuration conf = getTestConfiguration();
  final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
      .numDataNodes(3).build();
  try {
    final DistributedFileSystem fs = cluster.getFileSystem();
    final Path testFile = new Path("/testListLocatedStatus");
    final int blockSize = 4096;
    final int numBlocks = 10;
    // Create a test file
    final int repl = 2;
    DFSTestUtil.createFile(fs, testFile, blockSize, numBlocks * blockSize,
        blockSize, (short) repl, 0xADDED);
    DFSTestUtil.waitForReplication(fs, testFile, (short) repl, 30000);
    // Get the listing
    RemoteIterator<LocatedFileStatus> it = fs.listLocatedStatus(testFile);
    assertTrue("Expected file to be present", it.hasNext());
    LocatedFileStatus stat = it.next();
    BlockLocation[] locs = stat.getBlockLocations();
    assertEquals("Unexpected number of locations", numBlocks, locs.length);

    Set<String> dnStorageIds = new HashSet<>();
    for (DataNode d : cluster.getDataNodes()) {
      try (FsDatasetSpi.FsVolumeReferences volumes = d.getFSDataset()
          .getFsVolumeReferences()) {
        for (FsVolumeSpi vol : volumes) {
          dnStorageIds.add(vol.getStorageID());
        }
      }
    }

    for (BlockLocation loc : locs) {
      String[] ids = loc.getStorageIds();
      // Run it through a set to deduplicate, since there should be no dupes
      Set<String> storageIds = new HashSet<>();
      Collections.addAll(storageIds, ids);
      assertEquals("Unexpected num storage ids", repl, storageIds.size());
      // Make sure these are all valid storage IDs
      assertTrue("Unknown storage IDs found!", dnStorageIds.containsAll
          (storageIds));
      // Check storage types are the default, since we didn't set any
      StorageType[] types = loc.getStorageTypes();
      assertEquals("Unexpected num storage types", repl, types.length);
      for (StorageType t: types) {
        assertEquals("Unexpected storage type", StorageType.DEFAULT, t);
      }
    }
  } finally {
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}