Java FSConstants.LAYOUT_VERSION属性代码示例

/**
 * Safely reads the log version from the stream. Logic is exactly the same
 * as in the equivalent {@link EditLogFileInputStream} method.
 * @see EditLogFileInputStream#readLogVersion(DataInputStream)
 * @return The log version or 0 if stream is empty
 */
private static int readLogVersion(DataInputStream in) throws IOException {
  int logVersion = 0;
  in.mark(4);
  // See comments in EditLogFileInputStream as to why readLogVersion is
  // implemented in this way
  boolean available = true;
  try {
    logVersion = in.readByte();
  } catch (EOFException e) {
    available = false;
  }

  if (available) {
    in.reset();
    logVersion = in.readInt();
    if (logVersion < FSConstants.LAYOUT_VERSION) {
      throw new LedgerHeaderCorruptException(
          "Unexpected version of the log segment in the ledger: " + logVersion +
              ". Current version is " + FSConstants.LAYOUT_VERSION + ".");
    }
  }
  return logVersion;
}
 

@Test
public void testVerifyEditLogLedgerMetadata() throws Exception {
  EditLogLedgerMetadata m0 = new EditLogLedgerMetadata(
      FSConstants.LAYOUT_VERSION, 1, 1, 100);
  EditLogLedgerMetadata m1 = new EditLogLedgerMetadata(
      FSConstants.LAYOUT_VERSION, 2, 101, 200);

  String m0Path = manager.fullyQualifiedPathForLedger(m0);
  String m1Path = manager.fullyQualifiedPathForLedger(m1);
  manager.writeEditLogLedgerMetadata(m0Path, m0);
  manager.writeEditLogLedgerMetadata(m1Path, m1);

  assertTrue(m0 + " should verify under " + m0Path,
      manager.verifyEditLogLedgerMetadata(m0, m0Path));
  assertTrue(m1 + " should verify under " + m1Path,
      manager.verifyEditLogLedgerMetadata(m1, m1Path));

  assertFalse(m0 + " should not verify under " + m1Path,
      manager.verifyEditLogLedgerMetadata(m0, m1Path));
  assertFalse(m1 + " should not verify under" + m0Path,
      manager.verifyEditLogLedgerMetadata(m1, m0Path));

  assertFalse("Non-existent path should not verify!",
      manager.verifyEditLogLedgerMetadata(m0, "/does/not/exist"));
}
 

/**
 * Tests that {@link EditLogLedgerMetadata} can be correctly serialized
 * and deserialized.
 */
@Test
public void testReadAndWrite() throws Exception {
  EditLogLedgerMetadata ledgerMetadataIn = new EditLogLedgerMetadata(
      FSConstants.LAYOUT_VERSION, 1, 1, -1);
  EditLogLedgerMetadataWritable ledgerMetadataWritableIn =
      new EditLogLedgerMetadataWritable();
  ledgerMetadataWritableIn.set(ledgerMetadataIn);

  // Calls readWriteFields()
  byte[] editLogLedgerMedataBytes =
      WritableUtil.writableToByteArray(ledgerMetadataWritableIn);

  // Calls readFields()
  EditLogLedgerMetadataWritable ledgerMetadataWritableOut =
      WritableUtil.readWritableFromByteArray(editLogLedgerMedataBytes,
          new EditLogLedgerMetadataWritable());

  // Tests that deserialize(read(write(serialize(deserialize(m)) == m
  EditLogLedgerMetadata ledgerMetadataOut = ledgerMetadataWritableOut.get();
  assertEquals(ledgerMetadataIn, ledgerMetadataOut);
}
 

/**
 * Read the header of fsedit log
 * @param in fsedit stream
 * @return the edit log version number
 * @throws IOException if error occurs
 */
static int readLogVersion(DataInputStream in) throws IOException,
    LogHeaderCorruptException {
  int logVersion = 0;
  // Read log file version. Could be missing.
  in.mark(4);
  // If edits log is greater than 2G, available method will return negative
  // numbers, so we avoid having to call available
  boolean available = true;
  try {
    logVersion = in.readByte();
  } catch (EOFException e) {
    available = false;
  }

  if (available) {
    in.reset();
    logVersion = in.readInt();
    if (logVersion < FSConstants.LAYOUT_VERSION) { // future version
      throw new LogHeaderCorruptException(
          "Unexpected version of the file system log file: " + logVersion
              + ". Current version = " + FSConstants.LAYOUT_VERSION + ".");
    }
  }
  return logVersion;
}
 

/**
 * Format a namespace slice storage. 
 * @param sd the namespace storage
 * @param nsInfo the name space info
 * @throws IOException Signals that an I/O exception has occurred.
 */
private void format(StorageDirectory nsSdir, NamespaceInfo nsInfo) throws IOException {
  LOG.info("Formatting namespace " + namespaceID + " directory "
      + nsSdir.getCurrentDir());
  nsSdir.clearDirectory(); // create directory
  File rbwDir = new File(nsSdir.getCurrentDir(), STORAGE_DIR_RBW);
  File finalizedDir = new File(nsSdir.getCurrentDir(), STORAGE_DIR_FINALIZED);
  LOG.info("Creating Directories : " + rbwDir + ", " + finalizedDir);
  if (!rbwDir.mkdirs() || !finalizedDir.mkdirs()) {
    throw new IOException("Cannot create directories : " + rbwDir + ", "
        + finalizedDir);
  }
  this.layoutVersion = FSConstants.LAYOUT_VERSION;
  this.cTime = nsInfo.getCTime();
  this.namespaceID = nsInfo.getNamespaceID();
  this.storageType = NodeType.DATA_NODE;
  nsSdir.write();
}
 

/**
 * Determines if the given Namenode version and Datanode version
 * are compatible with each other. Compatibility in this case mean
 * that the Namenode and Datanode will successfully start up and
 * will work together. The rules for compatibility,
 * taken from the DFS Upgrade Design, are as follows:
 * <pre>
 * 1. The data-node does regular startup (no matter which options 
 *    it is started with) if
 *       softwareLV == storedLV AND 
 *       DataNode.FSSCTime == NameNode.FSSCTime
 * 2. The data-node performs an upgrade if it is started without any 
 *    options and
 *       |softwareLV| > |storedLV| OR 
 *       (softwareLV == storedLV AND
 *        DataNode.FSSCTime < NameNode.FSSCTime)
 * 3. NOT TESTED: The data-node rolls back if it is started with
 *    the -rollback option and
 *       |softwareLV| >= |previous.storedLV| AND 
 *       DataNode.previous.FSSCTime <= NameNode.FSSCTime
 * 4. In all other cases the startup fails.
 * </pre>
 */
boolean isVersionCompatible(StorageInfo namenodeVer, StorageInfo datanodeVer) {
  // check #0
  if (namenodeVer.getNamespaceID() != datanodeVer.getNamespaceID()) {
    LOG.info("namespaceIDs are not equal: isVersionCompatible=false");
    return false;
  }
  // check #1
  int softwareLV = FSConstants.LAYOUT_VERSION;  // will also be Namenode's LV
  int storedLV = datanodeVer.getLayoutVersion();
  if (softwareLV == storedLV &&  
      datanodeVer.getCTime() == namenodeVer.getCTime()) 
    {
      LOG.info("layoutVersions and cTimes are equal: isVersionCompatible=true");
      return true;
    }
  // check #2
  long absSoftwareLV = Math.abs((long)softwareLV);
  long absStoredLV = Math.abs((long)storedLV);
  if (absSoftwareLV > absStoredLV ||
      (softwareLV == storedLV &&
       datanodeVer.getCTime() < namenodeVer.getCTime())) 
    {
      LOG.info("softwareLayoutVersion is newer OR namenode cTime is newer: isVersionCompatible=true");
      return true;
    }
  // check #4
  LOG.info("default case: isVersionCompatible=false");
  return false;
}
 

private DatanodeRegistration getReg(long ctime, int namespaceId) {
  StorageInfo info = new StorageInfo();
  info.layoutVersion = FSConstants.LAYOUT_VERSION;
  info.cTime = ctime;
  info.namespaceID = namespaceId;
  DatanodeRegistration reg = new DatanodeRegistration();
  reg.setStorageInfo(info, null);
  return reg;
}
 

/**
 * Load an edit log, and continue applying the changes to the in-memory 
 * structure. This is where we ingest transactions into the standby.
 */
private int loadFSEdits() throws IOException {
  FSDirectory fsDir = fsNamesys.dir;
  int numEdits = 0;
  long startTime = FSNamesystem.now();
  LOG.info("Ingest: Consuming transactions: " + this.toString());

  try {
    logVersion = inputEditStream.getVersion();
    if (!LayoutVersion.supports(Feature.TXID_BASED_LAYOUT, logVersion))
      throw new RuntimeException("Log version is too old");
    
    currentPosition = inputEditStream.getPosition();
    numEdits = ingestFSEdits(); // continue to ingest 
  } finally {
    LOG.info("Ingest: Closing ingest for segment: " + this.toString());
    // At this time we are done reading the transaction log
    // We need to sync to have on disk status the same as in memory
    // if we saw end segment, we already synced
    if(endTxId == -1 && fsDir.fsImage.getEditLog().isOpen()) {
      fsDir.fsImage.getEditLog().logSync();
    }
    inputEditStream.close();
    standby.clearIngestState();
  }
  LOG.info("Ingest: Edits segment: " + this.toString()
      + " edits # " + numEdits 
      + " loaded in " + (FSNamesystem.now()-startTime)/1000 + " seconds.");

  if (logVersion != FSConstants.LAYOUT_VERSION) // other version
    numEdits++; // save this image asap
  return numEdits;
}
 

/**
 * Write all data storage files.
 * @throws IOException
 */
public void writeAll() throws IOException {
  this.layoutVersion = FSConstants.LAYOUT_VERSION;
  for (Iterator<StorageDirectory> it = storageDirs.iterator(); it.hasNext();) {
    it.next().write();
  }
}
 

/**
 * Format all available storage directories.
 */
public void format() throws IOException {
  this.layoutVersion = FSConstants.LAYOUT_VERSION;
  this.namespaceID = newNamespaceID();
  this.cTime = 0L;
  for (Iterator<StorageDirectory> it =
                         dirIterator(); it.hasNext();) {
    StorageDirectory sd = it.next();
    format(sd);
  }
}
 

/**
 * Read the header of fsedit log
 * @param in fsedit stream
 * @return the edit log version number
 * @throws IOException if error occurs
 */
static int readLogVersion(DataInputStream in) throws IOException,
    LogHeaderCorruptException {
  int logVersion = 0;
  // Read log file version. Could be missing.
  in.mark(4);
  // If edits log is greater than 2G, available method will return negative
  // numbers, so we avoid having to call available
  boolean available = true;
  try {
    logVersion = in.readByte();
  } catch (EOFException e) {
    available = false;
  }

  if (available) {
    in.reset();
    logVersion = in.readInt();
    if (logVersion < FSConstants.LAYOUT_VERSION) { // future version
      throw new LogHeaderCorruptException(
          "Unexpected version of the file system log file: " + logVersion
              + ". Current version = " + FSConstants.LAYOUT_VERSION + ".");
    }
  }
  if (logVersion < FSConstants.LAYOUT_VERSION || // future version
      logVersion > Storage.LAST_UPGRADABLE_LAYOUT_VERSION) { // unsupported
    throw new LogHeaderCorruptException(
        "Unexpected version of the file system log file: "
        + logVersion + ". Current version = "
        + FSConstants.LAYOUT_VERSION + ".");
  }
  return logVersion;
}
 

/**
 * Analyze storage directories.
 * Recover from previous transitions if required. 
 * Perform fs state transition if necessary depending on the namespace info.
 * Read storage info. 
 * 
 * @param nsInfo namespace information
 * @param dataDirs array of data storage directories
 * @param startOpt startup option
 * @throws IOException
 */
synchronized void recoverTransitionRead(DataNode datanode,
                           NamespaceInfo nsInfo,
                           Collection<File> dataDirs,
                           StartupOption startOpt
                           ) throws IOException {
  if (initialized) {
    // DN storage has been initialized, no need to do anything
    return;
  }

  if (FSConstants.LAYOUT_VERSION != nsInfo.getLayoutVersion()) {
    throw new IOException(
        "Data-node and name-node layout versions must be the same. Namenode LV: "
            + nsInfo.getLayoutVersion() + ", current LV: "
            + FSConstants.LAYOUT_VERSION);
  }
  
  // 1. For each data directory calculate its state and 
  // check whether all is consistent before transitioning.
  // Format and recover.
  analyzeStorageDirs(nsInfo, dataDirs, startOpt);

  // 2. Do transitions
  // Each storage directory is treated individually.
  // During startup some of them can upgrade or rollback
  // while others could be uptodate for the regular startup.
  doTransition(storageDirs, nsInfo, startOpt);

  // 3. make sure we have storage id set - if not - generate new one
  createStorageID(datanode.getPort());

  // 4. Update all storages. Some of them might have just been formatted.
  this.writeAll();

  this.initialized = true;
}
 

void format(StorageDirectory sd, NamespaceInfo nsInfo) throws IOException {
  sd.clearDirectory(); // create directory
  this.layoutVersion = FSConstants.LAYOUT_VERSION;
  this.namespaceID = nsInfo.getNamespaceID();  // mother namespaceid
  this.cTime = 0;
  // store storageID as it currently is
  sd.write();
}
 

/** Validate and set layout version from {@link Properties}*/
protected void setLayoutVersion(Properties props, StorageDirectory sd)
    throws IncorrectVersionException, InconsistentFSStateException {
  int lv = Integer.parseInt(getProperty(props, sd, LAYOUT_VERSION));
  if (lv < FSConstants.LAYOUT_VERSION) { // future version
    throw new IncorrectVersionException(lv, "storage directory "
        + sd.root.getAbsolutePath());
  }
  layoutVersion = lv;
}
 

@Override
protected void getFields(Properties props, StorageDirectory sd)
    throws IOException {
  setNamespaceID(props, sd);
  setcTime(props, sd);
  
  String snsid = props.getProperty(NAMESPACE_ID);
  setNameSpaceID(sd.getRoot(), snsid);

  String property = props.getProperty(LAYOUT_VERSION);
  int lv;
  if (property == null) {
    Integer topLayout = getTopLevelLayout(sd);
    if (topLayout == null) {
      throw new InconsistentFSStateException(sd.getRoot(),
          "Top level layout and NS level layout do not exist");
    }
    lv = topLayout;
  } else {
    lv = Integer.parseInt(property);
  }
  if (lv < FSConstants.LAYOUT_VERSION) { // future version
    throw new InconsistentFSStateException(sd.getRoot(),
        "has future layout version : " + lv);
  }
  layoutVersion = lv;
}