Java SecureIOUtils.openForRandomRead方法代码示例

import org.apache.hadoop.io.SecureIOUtils; //导入方法依赖的package包/类
protected ChannelFuture sendMapOutput(ChannelHandlerContext ctx, Channel ch,
    String user, String mapId, int reduce, MapOutputInfo mapOutputInfo)
    throws IOException {
  final TezIndexRecord info = mapOutputInfo.indexRecord;
  final ShuffleHeader header =
    new ShuffleHeader(mapId, info.getPartLength(), info.getRawLength(), reduce);
  final DataOutputBuffer dob = new DataOutputBuffer();
  header.write(dob);
  ch.write(wrappedBuffer(dob.getData(), 0, dob.getLength()));
  final File spillfile =
      new File(mapOutputInfo.mapOutputFileName.toString());
  RandomAccessFile spill;
  try {
    spill = SecureIOUtils.openForRandomRead(spillfile, "r", user, null);
  } catch (FileNotFoundException e) {
    LOG.info(spillfile + " not found");
    return null;
  }
  ChannelFuture writeFuture;
  final DefaultFileRegion partition =
      new DefaultFileRegion(spill.getChannel(), info.getStartOffset(), info.getPartLength());
  writeFuture = ch.write(partition);
  writeFuture.addListener(new ChannelFutureListener() {
    // TODO error handling; distinguish IO/connection failures,
    //      attribute to appropriate spill output
    @Override
    public void operationComplete(ChannelFuture future) {
      partition.releaseExternalResources();
    }
  });
  return writeFuture;
}
 

import org.apache.hadoop.io.SecureIOUtils; //导入方法依赖的package包/类
protected ChannelFuture sendMapOutput(ChannelHandlerContext ctx, Channel ch,
    String user, String mapId, int reduce, MapOutputInfo mapOutputInfo)
    throws IOException {
  final IndexRecord info = mapOutputInfo.indexRecord;
  final ShuffleHeader header =
    new ShuffleHeader(mapId, info.partLength, info.rawLength, reduce);
  final DataOutputBuffer dob = new DataOutputBuffer();
  header.write(dob);
  ch.write(wrappedBuffer(dob.getData(), 0, dob.getLength()));
  final File spillfile =
      new File(mapOutputInfo.mapOutputFileName.toString());
  RandomAccessFile spill;
  try {
    spill = SecureIOUtils.openForRandomRead(spillfile, "r", user, null);
  } catch (FileNotFoundException e) {
    LOG.info(spillfile + " not found");
    return null;
  }
  ChannelFuture writeFuture;
  if (ch.getPipeline().get(SslHandler.class) == null) {
    final FadvisedFileRegion partition = new FadvisedFileRegion(spill,
        info.startOffset, info.partLength, manageOsCache, readaheadLength,
        readaheadPool, spillfile.getAbsolutePath(), 
        shuffleBufferSize, shuffleTransferToAllowed);
    writeFuture = ch.write(partition);
    writeFuture.addListener(new ChannelFutureListener() {
        // TODO error handling; distinguish IO/connection failures,
        //      attribute to appropriate spill output
      @Override
      public void operationComplete(ChannelFuture future) {
        if (future.isSuccess()) {
          partition.transferSuccessful();
        }
        partition.releaseExternalResources();
      }
    });
  } else {
    // HTTPS cannot be done with zero copy.
    final FadvisedChunkedFile chunk = new FadvisedChunkedFile(spill,
        info.startOffset, info.partLength, sslFileBufferSize,
        manageOsCache, readaheadLength, readaheadPool,
        spillfile.getAbsolutePath());
    writeFuture = ch.write(chunk);
  }
  metrics.shuffleConnections.incr();
  metrics.shuffleOutputBytes.incr(info.partLength); // optimistic
  return writeFuture;
}
 

import org.apache.hadoop.io.SecureIOUtils; //导入方法依赖的package包/类
protected ChannelFuture sendMapOutput(ChannelHandlerContext ctx, Channel ch,
    String user, String jobId, String mapId, int reduce)
    throws IOException {
  // TODO replace w/ rsrc alloc
  // $x/$user/appcache/$appId/output/$mapId
  // TODO: Once Shuffle is out of NM, this can use MR APIs to convert between App and Job
  JobID jobID = JobID.forName(jobId);
  ApplicationId appID = ApplicationId.newInstance(
      Long.parseLong(jobID.getJtIdentifier()), jobID.getId());
  final String base =
      ContainerLocalizer.USERCACHE + "/" + user + "/"
          + ContainerLocalizer.APPCACHE + "/"
          + ConverterUtils.toString(appID) + "/output" + "/" + mapId;
  if (LOG.isDebugEnabled()) {
    LOG.debug("DEBUG0 " + base);
  }
  // Index file
  Path indexFileName = lDirAlloc.getLocalPathToRead(
      base + "/file.out.index", conf);
  // Map-output file
  Path mapOutputFileName = lDirAlloc.getLocalPathToRead(
      base + "/file.out", conf);
  if (LOG.isDebugEnabled()) {
    LOG.debug("DEBUG1 " + base + " : " + mapOutputFileName + " : "
        + indexFileName);
  }
  final IndexRecord info = 
    indexCache.getIndexInformation(mapId, reduce, indexFileName, user);
  final ShuffleHeader header =
    new ShuffleHeader(mapId, info.partLength, info.rawLength, reduce);
  final DataOutputBuffer dob = new DataOutputBuffer();
  header.write(dob);
  ch.write(wrappedBuffer(dob.getData(), 0, dob.getLength()));
  final File spillfile = new File(mapOutputFileName.toString());
  RandomAccessFile spill;
  try {
    spill = SecureIOUtils.openForRandomRead(spillfile, "r", user, null);
  } catch (FileNotFoundException e) {
    LOG.info(spillfile + " not found");
    return null;
  }
  ChannelFuture writeFuture;
  if (ch.getPipeline().get(SslHandler.class) == null) {
    final FadvisedFileRegion partition = new FadvisedFileRegion(spill,
        info.startOffset, info.partLength, manageOsCache, readaheadLength,
        readaheadPool, spillfile.getAbsolutePath());
    writeFuture = ch.write(partition);
    writeFuture.addListener(new ChannelFutureListener() {
        // TODO error handling; distinguish IO/connection failures,
        //      attribute to appropriate spill output
      @Override
      public void operationComplete(ChannelFuture future) {
        partition.releaseExternalResources();
      }
    });
  } else {
    // HTTPS cannot be done with zero copy.
    final FadvisedChunkedFile chunk = new FadvisedChunkedFile(spill,
        info.startOffset, info.partLength, sslFileBufferSize,
        manageOsCache, readaheadLength, readaheadPool,
        spillfile.getAbsolutePath());
    writeFuture = ch.write(chunk);
  }
  metrics.shuffleConnections.incr();
  metrics.shuffleOutputBytes.incr(info.partLength); // optimistic
  return writeFuture;
}
 

import org.apache.hadoop.io.SecureIOUtils; //导入方法依赖的package包/类
protected ChannelFuture sendMapOutput(ChannelHandlerContext ctx, Channel ch,
                                      String user, String mapId, Range reduceRange, MapOutputInfo outputInfo)
    throws IOException {
  TezIndexRecord firstIndex = null;
  TezIndexRecord lastIndex = null;

  DataOutputBuffer dobRange = new DataOutputBuffer();
  // Indicate how many record to be written
  WritableUtils.writeVInt(dobRange, reduceRange.getLast() - reduceRange.getFirst() + 1);
  ch.write(wrappedBuffer(dobRange.getData(), 0, dobRange.getLength()));
  for (int reduce = reduceRange.getFirst(); reduce <= reduceRange.getLast(); reduce++) {
    TezIndexRecord index = outputInfo.spillRecord.getIndex(reduce);
    // Records are only valid if they have a non-zero part length
    if (index.getPartLength() != 0) {
      if (firstIndex == null) {
        firstIndex = index;
      }
      lastIndex = index;
    }

    ShuffleHeader header = new ShuffleHeader(mapId, index.getPartLength(), index.getRawLength(), reduce);
    DataOutputBuffer dob = new DataOutputBuffer();
    header.write(dob);
    ch.write(wrappedBuffer(dob.getData(), 0, dob.getLength()));
  }

  final long rangeOffset = firstIndex.getStartOffset();
  final long rangePartLength = lastIndex.getStartOffset() + lastIndex.getPartLength() - firstIndex.getStartOffset();
  final File spillFile = new File(outputInfo.mapOutputFileName.toString());
  RandomAccessFile spill;
  try {
    spill = SecureIOUtils.openForRandomRead(spillFile, "r", user, null);
  } catch (FileNotFoundException e) {
    LOG.info(spillFile + " not found");
    return null;
  }
  ChannelFuture writeFuture;
  if (ch.getPipeline().get(SslHandler.class) == null) {
    final FadvisedFileRegion partition = new FadvisedFileRegion(spill,
        rangeOffset, rangePartLength, manageOsCache, readaheadLength,
        readaheadPool, spillFile.getAbsolutePath(),
        shuffleBufferSize, shuffleTransferToAllowed);
    writeFuture = ch.write(partition);
    writeFuture.addListener(new ChannelFutureListener() {
        // TODO error handling; distinguish IO/connection failures,
        //      attribute to appropriate spill output
      @Override
      public void operationComplete(ChannelFuture future) {
        if (future.isSuccess()) {
          partition.transferSuccessful();
        }
        partition.releaseExternalResources();
      }
    });
  } else {
    // HTTPS cannot be done with zero copy.
    final FadvisedChunkedFile chunk = new FadvisedChunkedFile(spill,
        rangeOffset, rangePartLength, sslFileBufferSize,
        manageOsCache, readaheadLength, readaheadPool,
        spillFile.getAbsolutePath());
    writeFuture = ch.write(chunk);
  }
  metrics.shuffleConnections.incr();
  metrics.shuffleOutputBytes.incr(rangePartLength); // optimistic
  return writeFuture;
}