Java HdfsConstants.INVALID_TXID属性代码示例

/**
 * @return the current state of the given segment, or null if the
 * segment does not exist.
 */
@VisibleForTesting
SegmentStateProto getSegmentInfo(long segmentTxId)
    throws IOException {
  EditLogFile elf = fjm.getLogFile(segmentTxId);
  if (elf == null) {
    return null;
  }
  if (elf.isInProgress()) {
    elf.scanLog();
  }
  if (elf.getLastTxId() == HdfsConstants.INVALID_TXID) {
    LOG.info("Edit log file " + elf + " appears to be empty. " +
        "Moving it aside...");
    elf.moveAsideEmptyFile();
    return null;
  }
  SegmentStateProto ret = SegmentStateProto.newBuilder()
      .setStartTxId(segmentTxId)
      .setEndTxId(elf.getLastTxId())
      .setIsInProgress(elf.isInProgress())
      .build();
  LOG.info("getSegmentInfo(" + segmentTxId + "): " + elf + " -> " +
      TextFormat.shortDebugString(ret));
  return ret;
}
 

static FSEditLogLoader.EditLogValidation validateEditLog(File file)
    throws IOException {
  EditLogFileInputStream in;
  try {
    in = new EditLogFileInputStream(file);
    in.getVersion(true); // causes us to read the header
  } catch (LogHeaderCorruptException e) {
    // If the header is malformed or the wrong value, this indicates a corruption
    LOG.warn("Log file " + file + " has no valid header", e);
    return new FSEditLogLoader.EditLogValidation(0,
        HdfsConstants.INVALID_TXID, true);
  }
  
  try {
    return FSEditLogLoader.validateEditLog(in);
  } finally {
    IOUtils.closeStream(in);
  }
}
 

/**
 * Scan the local storage directory, and return the segment containing
 * the highest transaction.
 * @return the EditLogFile with the highest transactions, or null
 * if no files exist.
 */
private synchronized EditLogFile scanStorageForLatestEdits() throws IOException {
  if (!fjm.getStorageDirectory().getCurrentDir().exists()) {
    return null;
  }
  
  LOG.info("Scanning storage " + fjm);
  List<EditLogFile> files = fjm.getLogFiles(0);
  
  while (!files.isEmpty()) {
    EditLogFile latestLog = files.remove(files.size() - 1);
    latestLog.scanLog();
    LOG.info("Latest log is " + latestLog);
    if (latestLog.getLastTxId() == HdfsConstants.INVALID_TXID) {
      // the log contains no transactions
      LOG.warn("Latest log " + latestLog + " has no transactions. " +
          "moving it aside and looking for previous log");
      latestLog.moveAsideEmptyFile();
    } else {
      return latestLog;
    }
  }
  
  LOG.info("No files in " + fjm);
  return null;
}
 

/**
 * Find the last valid transaction ID in the stream.
 * If there are invalid or corrupt transactions in the middle of the stream,
 * validateEditLog will skip over them.
 * This reads through the stream but does not close it.
 */
static EditLogValidation validateEditLog(EditLogInputStream in) {
  long lastPos = 0;
  long lastTxId = HdfsConstants.INVALID_TXID;
  long numValid = 0;
  FSEditLogOp op = null;
  while (true) {
    lastPos = in.getPosition();
    try {
      if ((op = in.readOp()) == null) {
        break;
      }
    } catch (Throwable t) {
      FSImage.LOG.warn("Caught exception after reading " + numValid +
          " ops from " + in + " while determining its valid length." +
          "Position was " + lastPos, t);
      in.resync();
      FSImage.LOG.warn("After resync, position is " + in.getPosition());
      continue;
    }
    if (lastTxId == HdfsConstants.INVALID_TXID
        || op.getTransactionId() > lastTxId) {
      lastTxId = op.getTransactionId();
    }
    numValid++;
  }
  return new EditLogValidation(lastPos, lastTxId, false);
}
 

private RequestInfo convert(
    QJournalProtocolProtos.RequestInfoProto reqInfo) {
  return new RequestInfo(
      reqInfo.getJournalId().getIdentifier(),
      reqInfo.getEpoch(),
      reqInfo.getIpcSerialNumber(),
      reqInfo.hasCommittedTxId() ?
        reqInfo.getCommittedTxId() : HdfsConstants.INVALID_TXID);
}
 

/**
 * Reload any data that may have been cached. This is necessary
 * when we first load the Journal, but also after any formatting
 * operation, since the cached data is no longer relevant.
 */
private synchronized void refreshCachedData() {
  IOUtils.closeStream(committedTxnId);
  
  File currentDir = storage.getSingularStorageDir().getCurrentDir();
  this.lastPromisedEpoch = new PersistentLongFile(
      new File(currentDir, LAST_PROMISED_FILENAME), 0);
  this.lastWriterEpoch = new PersistentLongFile(
      new File(currentDir, LAST_WRITER_EPOCH), 0);
  this.committedTxnId = new BestEffortLongFile(
      new File(currentDir, COMMITTED_TXID_FILENAME),
      HdfsConstants.INVALID_TXID);
}
 

/**
 * Similar with decodeOp(), but instead of doing the real decoding, we skip
 * the content of the op if the length of the editlog is supported.
 * @return the last txid of the segment, or INVALID_TXID on exception
 */
public long scanOp() throws IOException {
  if (supportEditLogLength) {
    limiter.setLimit(maxOpSize);
    in.mark(maxOpSize);

    final byte opCodeByte;
    try {
      opCodeByte = in.readByte(); // op code
    } catch (EOFException e) {
      return HdfsConstants.INVALID_TXID;
    }

    FSEditLogOpCodes opCode = FSEditLogOpCodes.fromByte(opCodeByte);
    if (opCode == OP_INVALID) {
      verifyTerminator();
      return HdfsConstants.INVALID_TXID;
    }

    int length = in.readInt(); // read the length of the op
    long txid = in.readLong(); // read the txid

    // skip the remaining content
    IOUtils.skipFully(in, length - 8); 
    // TODO: do we want to verify checksum for JN? For now we don't.
    return txid;
  } else {
    FSEditLogOp op = decodeOp();
    return op == null ? HdfsConstants.INVALID_TXID : op.getTransactionId();
  }
}
 

EditLogLedgerMetadata(String zkPath, int dataLayoutVersion,
                      long ledgerId, long firstTxId) {
  this.zkPath = zkPath;
  this.dataLayoutVersion = dataLayoutVersion;
  this.ledgerId = ledgerId;
  this.firstTxId = firstTxId;
  this.lastTxId = HdfsConstants.INVALID_TXID;
  this.inprogress = true;
}
 

@Override
List<FSImageFile> getLatestImages() throws IOException {
  // We should have at least one image and one edits dirs
  if (latestNameSD == null)
    throw new IOException("Image file is not found in " + imageDirs);
  if (latestEditsSD == null)
    throw new IOException("Edits file is not found in " + editsDirs);
  
  // Make sure we are loading image and edits from same checkpoint
  if (latestNameCheckpointTime > latestEditsCheckpointTime
      && latestNameSD != latestEditsSD
      && latestNameSD.getStorageDirType() == NameNodeDirType.IMAGE
      && latestEditsSD.getStorageDirType() == NameNodeDirType.EDITS) {
    // This is a rare failure when NN has image-only and edits-only
    // storage directories, and fails right after saving images,
    // in some of the storage directories, but before purging edits.
    // See -NOTE- in saveNamespace().
    LOG.error("This is a rare failure scenario!!!");
    LOG.error("Image checkpoint time " + latestNameCheckpointTime +
              " > edits checkpoint time " + latestEditsCheckpointTime);
    LOG.error("Name-node will treat the image as the latest state of " +
              "the namespace. Old edits will be discarded.");
  } else if (latestNameCheckpointTime != latestEditsCheckpointTime) {
    throw new IOException("Inconsistent storage detected, " +
                    "image and edits checkpoint times do not match. " +
                    "image checkpoint time = " + latestNameCheckpointTime +
                    "edits checkpoint time = " + latestEditsCheckpointTime);
  }

  needToSaveAfterRecovery = doRecovery();
  
  FSImageFile file = new FSImageFile(latestNameSD, 
      NNStorage.getStorageFile(latestNameSD, NameNodeFile.IMAGE),
      HdfsConstants.INVALID_TXID);
  LinkedList<FSImageFile> ret = new LinkedList<FSImageFile>();
  ret.add(file);
  return ret;
}
 

RedundantEditLogInputStream(Collection<EditLogInputStream> streams,
    long startTxId) {
  this.curIdx = 0;
  this.prevTxId = (startTxId == HdfsConstants.INVALID_TXID) ?
    HdfsConstants.INVALID_TXID : (startTxId - 1);
  this.state = (streams.isEmpty()) ? State.EOF : State.SKIP_UNTIL;
  this.prevException = null;
  // EditLogInputStreams in a RedundantEditLogInputStream must be finalized,
  // and can't be pre-transactional.
  EditLogInputStream first = null;
  for (EditLogInputStream s : streams) {
    Preconditions.checkArgument(s.getFirstTxId() !=
        HdfsConstants.INVALID_TXID, "invalid first txid in stream: %s", s);
    Preconditions.checkArgument(s.getLastTxId() !=
        HdfsConstants.INVALID_TXID, "invalid last txid in stream: %s", s);
    if (first == null) {
      first = s;
    } else {
      Preconditions.checkArgument(s.getFirstTxId() == first.getFirstTxId(),
        "All streams in the RedundantEditLogInputStream must have the same " +
        "start transaction ID!  " + first + " had start txId " +
        first.getFirstTxId() + ", but " + s + " had start txId " +
        s.getFirstTxId());
    }
  }

  this.streams = streams.toArray(new EditLogInputStream[0]);

  // We sort the streams here so that the streams that end later come first.
  Arrays.sort(this.streams, new Comparator<EditLogInputStream>() {
    @Override
    public int compare(EditLogInputStream a, EditLogInputStream b) {
      return Longs.compare(b.getLastTxId(), a.getLastTxId());
    }
  });
}
 

@Override
public Long apply(RemoteEditLog log) {
  if (null == log) {
    return HdfsConstants.INVALID_TXID;
  }
  return log.getStartTxId();
}
 

@Test
public void testPreserveEditLogs() throws Exception {
  conf = new HdfsConfiguration();
  conf = UpgradeUtilities.initializeStorageStateConf(1, conf);
  String[] nameNodeDirs = conf.getStrings(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY);
  conf.setBoolean(DFSConfigKeys.DFS_DATANODE_DUPLICATE_REPLICA_DELETION, false);

  log("Normal NameNode upgrade", 1);
  File[] created =
      UpgradeUtilities.createNameNodeStorageDirs(nameNodeDirs, "current");
  for (final File createdDir : created) {
    List<String> fileNameList =
        IOUtils.listDirectory(createdDir, EditLogsFilter.INSTANCE);
    for (String fileName : fileNameList) {
      String tmpFileName = fileName + ".tmp";
      File existingFile = new File(createdDir, fileName);
      File tmpFile = new File(createdDir, tmpFileName);
      Files.move(existingFile.toPath(), tmpFile.toPath());
      File newFile = new File(createdDir, fileName);
      Preconditions.checkState(newFile.createNewFile(),
          "Cannot create new edits log file in " + createdDir);
      EditLogFileInputStream in = new EditLogFileInputStream(tmpFile,
          HdfsConstants.INVALID_TXID, HdfsConstants.INVALID_TXID,
          false);
      EditLogFileOutputStream out = new EditLogFileOutputStream(conf, newFile,
          (int)tmpFile.length());
      out.create(NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION + 1);
      FSEditLogOp logOp = in.readOp();
      while (logOp != null) {
        out.write(logOp);
        logOp = in.readOp();
      }
      out.setReadyToFlush();
      out.flushAndSync(true);
      out.close();
      Files.delete(tmpFile.toPath());
    }
  }

  cluster = createCluster();

  DFSInotifyEventInputStream ieis =
      cluster.getFileSystem().getInotifyEventStream(0);
  EventBatch batch = ieis.poll();
  Event[] events = batch.getEvents();
  assertTrue("Should be able to get transactions before the upgrade.",
      events.length > 0);
  assertEquals(events[0].getEventType(), Event.EventType.CREATE);
  assertEquals(((CreateEvent) events[0]).getPath(), "/TestUpgrade");
  cluster.shutdown();
  UpgradeUtilities.createEmptyDirs(nameNodeDirs);
}
 

public String getTransactionIdStr() {
  return (txid == HdfsConstants.INVALID_TXID) ? "(none)" : "" + txid;
}
 

/**
 * Go through the next operation from the stream storage.
 * @return the txid of the next operation.
 */
protected long scanNextOp() throws IOException {
  FSEditLogOp next = readOp();
  return next != null ? next.txid : HdfsConstants.INVALID_TXID;
}
 

@Override
public void selectInputStreams(Collection<EditLogInputStream> streams,
    long fromTxId, boolean inProgressOk)
    throws IOException {
  List<EditLogLedgerMetadata> currentLedgerList = getLedgerList(fromTxId,
      inProgressOk);
  try {
    BookKeeperEditLogInputStream elis = null;
    for (EditLogLedgerMetadata l : currentLedgerList) {
      long lastTxId = l.getLastTxId();
      if (l.isInProgress()) {
        lastTxId = recoverLastTxId(l, false);
      }
      // Check once again, required in case of InProgress and is case of any
      // gap.
      if (fromTxId >= l.getFirstTxId() && fromTxId <= lastTxId) {
        LedgerHandle h;
        if (l.isInProgress()) { // we don't want to fence the current journal
          h = bkc.openLedgerNoRecovery(l.getLedgerId(),
              BookKeeper.DigestType.MAC, digestpw.getBytes(Charsets.UTF_8));
        } else {
          h = bkc.openLedger(l.getLedgerId(), BookKeeper.DigestType.MAC,
              digestpw.getBytes(Charsets.UTF_8));
        }
        elis = new BookKeeperEditLogInputStream(h, l);
        elis.skipTo(fromTxId);
      } else {
        // If mismatches then there might be some gap, so we should not check
        // further.
        return;
      }
      streams.add(elis);
      if (elis.getLastTxId() == HdfsConstants.INVALID_TXID) {
        return;
      }
      fromTxId = elis.getLastTxId() + 1;
    }
  } catch (BKException e) {
    throw new IOException("Could not open ledger for " + fromTxId, e);
  } catch (InterruptedException ie) {
    Thread.currentThread().interrupt();
    throw new IOException("Interrupted opening ledger for " + fromTxId, ie);
  }
}