Java DirectoryListing.getPartialListing方法代码示例

import org.apache.hadoop.hdfs.protocol.DirectoryListing; //导入方法依赖的package包/类
void verifyFile(final Path file, final Byte expectedPolicyId)
    throws Exception {
  final Path parent = file.getParent();
  DirectoryListing children = dfs.getClient().listPaths(
      parent.toString(), HdfsFileStatus.EMPTY_NAME, true);
  for (HdfsFileStatus child : children.getPartialListing()) {
    if (child.getLocalName().equals(file.getName())) {
      verifyFile(parent,  child, expectedPolicyId);
      return;
    }
  }
  Assert.fail("File " + file + " not found.");
}
 

import org.apache.hadoop.hdfs.protocol.DirectoryListing; //导入方法依赖的package包/类
static void recoverAllLeases(DFSClient dfs, 
    Path path) throws IOException {
  String pathStr = path.toString();
  HdfsFileStatus status = dfs.getFileInfo(pathStr);
  if (!status.isDir()) {
    dfs.recoverLease(pathStr);
    return;
  }
  byte prev[] = HdfsFileStatus.EMPTY_NAME;
  DirectoryListing dirList;
  do {
    dirList = dfs.listPaths(pathStr, prev);
    HdfsFileStatus files[] = dirList.getPartialListing();
    for (HdfsFileStatus f : files) {
      recoverAllLeases(dfs, f.getFullPath(path));
    }
    prev = dirList.getLastName();
  } while (dirList.hasMore());
}
 

import org.apache.hadoop.hdfs.protocol.DirectoryListing; //导入方法依赖的package包/类
/**
 * Retrieve list of files under @hdfsDir for @hdfsClient.
 */
public static List<HdfsFileStatus> getHdfsFileList(DFSClient hdfsClient,
                                                   String hdfsDir)
    throws IOException {
  List<HdfsFileStatus> fileList = Lists.newArrayList();
  // Build a list of files.
  DirectoryListing listing = null;
  String continuation = "";
  while (true) {
    listing = hdfsClient.listPaths(hdfsDir, continuation.getBytes());
    for (HdfsFileStatus fileStatus : listing.getPartialListing()) {
      fileList.add(fileStatus);
    }
    // Go through the listing and paginate.
    if (!listing.hasMore()) {
      break;
    } else {
      continuation = new String(listing.getLastName());
    }
  }
  return fileList;
}
 

import org.apache.hadoop.hdfs.protocol.DirectoryListing; //导入方法依赖的package包/类
private void verifyRecursively(final Path parent,
    final HdfsFileStatus status) throws Exception {
  if (status.isDir()) {
    Path fullPath = parent == null ?
        new Path("/") : status.getFullPath(parent);
    DirectoryListing children = dfs.getClient().listPaths(
        fullPath.toString(), HdfsFileStatus.EMPTY_NAME, true);
    for (HdfsFileStatus child : children.getPartialListing()) {
      verifyRecursively(fullPath, child);
    }
  } else if (!status.isSymlink()) { // is file
    verifyFile(parent, status, null);
  }
}
 

import org.apache.hadoop.hdfs.protocol.DirectoryListing; //导入方法依赖的package包/类
private static StreamingOutput getListingStream(final NamenodeProtocols np,
    final String p) throws IOException {
  final DirectoryListing first =
      getDirectoryListing(np, p, HdfsFileStatus.EMPTY_NAME);

  return new StreamingOutput() {
    @Override
    public void write(final OutputStream outstream) throws IOException {
      final PrintWriter out =
          new PrintWriter(new OutputStreamWriter(outstream, Charsets.UTF_8));
      out.println("{\"" + FileStatus.class.getSimpleName() + "es\":{\"" +
          FileStatus.class.getSimpleName() + "\":[");

      final HdfsFileStatus[] partial = first.getPartialListing();
      if (partial.length > 0) {
        out.print(JsonUtil.toJsonString(partial[0], false));
      }
      for (int i = 1; i < partial.length; i++) {
        out.println(',');
        out.print(JsonUtil.toJsonString(partial[i], false));
      }

      for (DirectoryListing curr = first; curr.hasMore(); ) {
        curr = getDirectoryListing(np, p, curr.getLastName());
        for (HdfsFileStatus s : curr.getPartialListing()) {
          out.println(',');
          out.print(JsonUtil.toJsonString(s, false));
        }
      }
      
      out.println();
      out.println("]}}");
      out.flush();
    }
  };
}
 

import org.apache.hadoop.hdfs.protocol.DirectoryListing; //导入方法依赖的package包/类
private static StreamingOutput getListingStream(final NameNode np, 
    final String p) throws IOException {
  final DirectoryListing first = getDirectoryListing(np, p,
      HdfsFileStatus.EMPTY_NAME);

  return new StreamingOutput() {
    @Override
    public void write(final OutputStream outstream) throws IOException {
      final PrintStream out = new PrintStream(outstream);
      out.println("{\"" + FileStatus.class.getSimpleName() + "es\":{\""
          + FileStatus.class.getSimpleName() + "\":[");

      final HdfsFileStatus[] partial = first.getPartialListing();
      if (partial.length > 0) {
        out.print(JsonUtil.toJsonString(partial[0], false));
      }
      for(int i = 1; i < partial.length; i++) {
        out.println(',');
        out.print(JsonUtil.toJsonString(partial[i], false));
      }

      for(DirectoryListing curr = first; curr.hasMore(); ) { 
        curr = getDirectoryListing(np, p, curr.getLastName());
        for(HdfsFileStatus s : curr.getPartialListing()) {
          out.println(',');
          out.print(JsonUtil.toJsonString(s, false));
        }
      }
      
      out.println();
      out.println("]}}");
    }
  };
}