Java RemoteEditLogManifest类代码示例

import org.apache.hadoop.hdfs.server.protocol.RemoteEditLogManifest; //导入依赖的package包/类
static void rollForwardByApplyingLogs(
    RemoteEditLogManifest manifest,
    FSImage dstImage,
    FSNamesystem dstNamesystem) throws IOException {
  NNStorage dstStorage = dstImage.getStorage();

  List<EditLogInputStream> editsStreams = Lists.newArrayList();    
  for (RemoteEditLog log : manifest.getLogs()) {
    if (log.getEndTxId() > dstImage.getLastAppliedTxId()) {
      File f = dstStorage.findFinalizedEditsFile(
          log.getStartTxId(), log.getEndTxId());
      editsStreams.add(new EditLogFileInputStream(f, log.getStartTxId(), 
                                                  log.getEndTxId(), true));
    }
  }
  LOG.info("Checkpointer about to load edits from " +
      editsStreams.size() + " stream(s).");
  dstImage.loadEdits(editsStreams, dstNamesystem);
}
 

import org.apache.hadoop.hdfs.server.protocol.RemoteEditLogManifest; //导入依赖的package包/类
/**
 * @see QJournalProtocol#getEditLogManifest(String, long, boolean)
 */
public RemoteEditLogManifest getEditLogManifest(long sinceTxId,
    boolean inProgressOk) throws IOException {
  // No need to checkRequest() here - anyone may ask for the list
  // of segments.
  checkFormatted();
  
  List<RemoteEditLog> logs = fjm.getRemoteEditLogs(sinceTxId, inProgressOk);
  
  if (inProgressOk) {
    RemoteEditLog log = null;
    for (Iterator<RemoteEditLog> iter = logs.iterator(); iter.hasNext();) {
      log = iter.next();
      if (log.isInProgress()) {
        iter.remove();
        break;
      }
    }
    if (log != null && log.isInProgress()) {
      logs.add(new RemoteEditLog(log.getStartTxId(), getHighestWrittenTxId(),
          true));
    }
  }
  
  return new RemoteEditLogManifest(logs);
}
 

import org.apache.hadoop.hdfs.server.protocol.RemoteEditLogManifest; //导入依赖的package包/类
static void rollForwardByApplyingLogs(
    RemoteEditLogManifest manifest,
    FSImage dstImage) throws IOException {
  NNStorage dstStorage = dstImage.storage;

  List<EditLogInputStream> editsStreams = new ArrayList<EditLogInputStream>();    
  for (RemoteEditLog log : manifest.getLogs()) {
    if (log.inProgress())
      break;
    File f = dstStorage.findFinalizedEditsFile(
        log.getStartTxId(), log.getEndTxId());
    if (log.getStartTxId() > dstImage.getLastAppliedTxId()) {
      editsStreams.add(new EditLogFileInputStream(f, log.getStartTxId(), 
                                                  log.getEndTxId(), false));
     }
  }
  dstImage.loadEdits(editsStreams);
}
 

import org.apache.hadoop.hdfs.server.protocol.RemoteEditLogManifest; //导入依赖的package包/类
/**
 * Merge image and edits, and verify consistency with the signature.
 */
private void doMerge(CheckpointSignature sig,
    RemoteEditLogManifest manifest, boolean loadImage) throws IOException {
  NNStorage dstStorage = this.storage;
  dstStorage.setStorageInfo(sig);

  if (loadImage) {
    File file = dstStorage.findImageFile(sig.mostRecentCheckpointTxId);
    if (file == null) {
      throw new IOException("Couldn't find image file at txid "
          + sig.mostRecentCheckpointTxId + " even though it should have "
          + "just been downloaded");
    }
    loadFSImage(new ImageInputStream(sig.mostRecentCheckpointTxId,
        new FileInputStream(file), null, "image", file.length()));
  }
  FSImage.rollForwardByApplyingLogs(manifest, this);
  this.saveFSImageInAllDirs(this.getLastAppliedTxId(), false);
  dstStorage.writeAll();
}
 

import org.apache.hadoop.hdfs.server.protocol.RemoteEditLogManifest; //导入依赖的package包/类
@Override
public GetEditLogManifestResponseProto getEditLogManifest(byte[] jid,
    long sinceTxId) throws IOException {
  
  RemoteEditLogManifest manifest = jn.getOrCreateJournal(jid)
      .getEditLogManifest(sinceTxId);
  
  GetEditLogManifestResponseProto ret = new GetEditLogManifestResponseProto();
  ret.setLogs(manifest.getLogs());
  ret.setHttpPort(jn.getBoundHttpAddress().getPort());
  
  if (JournalNode.LOG.isDebugEnabled()) {
    JournalNode.LOG.info("Returning manifest " + manifest.toString());
  }

  return ret;
}
 

import org.apache.hadoop.hdfs.server.protocol.RemoteEditLogManifest; //导入依赖的package包/类
static void rollForwardByApplyingLogs(
    RemoteEditLogManifest manifest,
    FSImage dstImage,
    FSNamesystem dstNamesystem) throws IOException {
  NNStorage dstStorage = dstImage.getStorage();

  List<EditLogInputStream> editsStreams = Lists.newArrayList();    
  for (RemoteEditLog log : manifest.getLogs()) {
    if (log.getEndTxId() > dstImage.getLastAppliedTxId()) {
      File f = dstStorage.findFinalizedEditsFile(
          log.getStartTxId(), log.getEndTxId());
      editsStreams.add(new EditLogFileInputStream(f, log.getStartTxId(), 
                                                  log.getEndTxId(), true));
    }
  }
  LOG.info("Checkpointer about to load edits from " +
      editsStreams.size() + " stream(s).");
  dstImage.loadEdits(editsStreams, dstNamesystem, null);
}
 

import org.apache.hadoop.hdfs.server.protocol.RemoteEditLogManifest; //导入依赖的package包/类
static void doMerge(
    CheckpointSignature sig, RemoteEditLogManifest manifest,
    boolean loadImage, FSImage dstImage, FSNamesystem dstNamesystem)
    throws IOException {   
  NNStorage dstStorage = dstImage.getStorage();
  
  dstStorage.setStorageInfo(sig);
  if (loadImage) {
    File file = dstStorage.findImageFile(sig.mostRecentCheckpointTxId);
    if (file == null) {
      throw new IOException("Couldn't find image file at txid " + 
          sig.mostRecentCheckpointTxId + " even though it should have " +
          "just been downloaded");
    }
    dstImage.reloadFromImageFile(file, dstNamesystem);
    dstNamesystem.dir.imageLoadComplete();
  }
  // error simulation code for junit test
  CheckpointFaultInjector.getInstance().duringMerge();   

  Checkpointer.rollForwardByApplyingLogs(manifest, dstImage, dstNamesystem);
  // The following has the side effect of purging old fsimages/edit logs.
  dstImage.saveFSImageInAllDirs(dstNamesystem, dstImage.getLastAppliedTxId());
  dstStorage.writeAll();
}
 

import org.apache.hadoop.hdfs.server.protocol.RemoteEditLogManifest; //导入依赖的package包/类
@Override
public ListenableFuture<RemoteEditLogManifest> getEditLogManifest(
    final long fromTxnId, final boolean forReading, 
    final boolean inProgressOk) {
  return executor.submit(new Callable<RemoteEditLogManifest>() {
    @Override
    public RemoteEditLogManifest call() throws IOException {
      GetEditLogManifestResponseProto ret = getProxy().getEditLogManifest(
          journalId, fromTxnId, forReading, inProgressOk);
      // Update the http port, since we need this to build URLs to any of the
      // returned logs.
      httpPort = ret.getHttpPort();
      return PBHelper.convert(ret.getManifest());
    }
  });
}
 

import org.apache.hadoop.hdfs.server.protocol.RemoteEditLogManifest; //导入依赖的package包/类
/**
 * @see QJournalProtocol#getEditLogManifest(String, long, boolean)
 */
public RemoteEditLogManifest getEditLogManifest(long sinceTxId,
    boolean inProgressOk) throws IOException {
  // No need to checkRequest() here - anyone may ask for the list
  // of segments.
  checkFormatted();
  
  List<RemoteEditLog> logs = fjm.getRemoteEditLogs(sinceTxId, inProgressOk);
  
  if (inProgressOk) {
    RemoteEditLog log = null;
    for (Iterator<RemoteEditLog> iter = logs.iterator(); iter.hasNext();) {
      log = iter.next();
      if (log.isInProgress()) {
        iter.remove();
        break;
      }
    }
    if (log != null && log.isInProgress()) {
      logs.add(new RemoteEditLog(log.getStartTxId(), getHighestWrittenTxId()));
    }
  }
  
  return new RemoteEditLogManifest(logs);
}
 

import org.apache.hadoop.hdfs.server.protocol.RemoteEditLogManifest; //导入依赖的package包/类
@Override // NamenodeProtocol
public RemoteEditLogManifest getEditLogManifest(long sinceTxId)
    throws IOException {
  checkNNStartup();
  namesystem.checkOperation(OperationCategory.READ);
  namesystem.checkSuperuserPrivilege();
  return namesystem.getEditLog().getEditLogManifest(sinceTxId);
}
 

import org.apache.hadoop.hdfs.server.protocol.RemoteEditLogManifest; //导入依赖的package包/类
static void doMerge(
    CheckpointSignature sig, RemoteEditLogManifest manifest,
    boolean loadImage, FSImage dstImage, FSNamesystem dstNamesystem)
    throws IOException {   
  NNStorage dstStorage = dstImage.getStorage();
  
  dstStorage.setStorageInfo(sig);
  if (loadImage) {
    File file = dstStorage.findImageFile(NameNodeFile.IMAGE,
        sig.mostRecentCheckpointTxId);
    if (file == null) {
      throw new IOException("Couldn't find image file at txid " + 
          sig.mostRecentCheckpointTxId + " even though it should have " +
          "just been downloaded");
    }
    dstNamesystem.writeLock();
    try {
      dstImage.reloadFromImageFile(file, dstNamesystem);
    } finally {
      dstNamesystem.writeUnlock();
    }
    dstNamesystem.imageLoadComplete();
  }
  // error simulation code for junit test
  CheckpointFaultInjector.getInstance().duringMerge();   

  Checkpointer.rollForwardByApplyingLogs(manifest, dstImage, dstNamesystem);
  // The following has the side effect of purging old fsimages/edit logs.
  dstImage.saveFSImageInAllDirs(dstNamesystem, dstImage.getLastAppliedTxId());
  dstStorage.writeAll();
}
 

import org.apache.hadoop.hdfs.server.protocol.RemoteEditLogManifest; //导入依赖的package包/类
public static RemoteEditLogManifestProto convert(
    RemoteEditLogManifest manifest) {
  RemoteEditLogManifestProto.Builder builder = RemoteEditLogManifestProto
      .newBuilder();
  for (RemoteEditLog log : manifest.getLogs()) {
    builder.addLogs(convert(log));
  }
  return builder.build();
}
 

import org.apache.hadoop.hdfs.server.protocol.RemoteEditLogManifest; //导入依赖的package包/类
@Override
public GetEditLogManifestResponseProto getEditLogManifest(
    RpcController unused, GetEditLogManifestRequestProto request)
    throws ServiceException {
  RemoteEditLogManifest manifest;
  try {
    manifest = impl.getEditLogManifest(request.getSinceTxId());
  } catch (IOException e) {
    throw new ServiceException(e);
  }
  return GetEditLogManifestResponseProto.newBuilder()
      .setManifest(PBHelper.convert(manifest)).build();
}
 

import org.apache.hadoop.hdfs.server.protocol.RemoteEditLogManifest; //导入依赖的package包/类
@Override
public RemoteEditLogManifest getEditLogManifest(long sinceTxId)
    throws IOException {
  GetEditLogManifestRequestProto req = GetEditLogManifestRequestProto
      .newBuilder().setSinceTxId(sinceTxId).build();
  try {
    return PBHelper.convert(rpcProxy.getEditLogManifest(NULL_CONTROLLER, req)
        .getManifest());
  } catch (ServiceException e) {
    throw ProtobufHelper.getRemoteException(e);
  }
}
 

import org.apache.hadoop.hdfs.server.protocol.RemoteEditLogManifest; //导入依赖的package包/类
public QuorumCall<AsyncLogger, RemoteEditLogManifest> getEditLogManifest(
    long fromTxnId, boolean inProgressOk) {
  Map<AsyncLogger,
      ListenableFuture<RemoteEditLogManifest>> calls
      = Maps.newHashMap();
  for (AsyncLogger logger : loggers) {
    ListenableFuture<RemoteEditLogManifest> future =
        logger.getEditLogManifest(fromTxnId, inProgressOk);
    calls.put(logger, future);
  }
  return QuorumCall.create(calls);
}