Java BlockPoolSliceStorage类代码示例

import org.apache.hadoop.hdfs.server.datanode.BlockPoolSliceStorage; //导入依赖的package包/类
/**
 * Create a <code>version</code> file for datanode inside the specified parent
 * directory.  If such a file already exists, it will be overwritten.
 * The given version string will be written to the file as the layout
 * version. None of the parameters may be null.
 *
 * @param parent directory where namenode VERSION file is stored
 * @param version StorageInfo to create VERSION file from
 * @param bpid Block pool Id
 * @param bpidToWrite Block pool Id to write into the version file
 */
public static void createDataNodeVersionFile(File[] parent,
    StorageInfo version, String bpid, String bpidToWrite) throws IOException {
  DataStorage storage = new DataStorage(version);
  storage.setDatanodeUuid("FixedDatanodeUuid");

  File[] versionFiles = new File[parent.length];
  for (int i = 0; i < parent.length; i++) {
    File versionFile = new File(parent[i], "VERSION");
    StorageDirectory sd = new StorageDirectory(parent[i].getParentFile());
    storage.createStorageID(sd, false);
    storage.writeProperties(versionFile, sd);
    versionFiles[i] = versionFile;
    File bpDir = BlockPoolSliceStorage.getBpRoot(bpid, parent[i]);
    createBlockPoolVersionFile(bpDir, version, bpidToWrite);
  }
}
 

import org.apache.hadoop.hdfs.server.datanode.BlockPoolSliceStorage; //导入依赖的package包/类
/**
 * Create a <code>version</code> file for datanode inside the specified parent
 * directory.  If such a file already exists, it will be overwritten.
 * The given version string will be written to the file as the layout
 * version. None of the parameters may be null.
 *
 * @param parent directory where namenode VERSION file is stored
 * @param version StorageInfo to create VERSION file from
 * @param bpid Block pool Id
 * @param bpidToWrite Block pool Id to write into the version file
 */
public static void createDataNodeVersionFile(File[] parent,
    StorageInfo version, String bpid, String bpidToWrite) throws IOException {
  DataStorage storage = new DataStorage(version);
  storage.setDatanodeUuid("FixedDatanodeUuid");

  File[] versionFiles = new File[parent.length];
  for (int i = 0; i < parent.length; i++) {
    File versionFile = new File(parent[i], "VERSION");
    StorageDirectory sd = new StorageDirectory(parent[i].getParentFile());
    storage.createStorageID(sd);
    storage.writeProperties(versionFile, sd);
    versionFiles[i] = versionFile;
    File bpDir = BlockPoolSliceStorage.getBpRoot(bpid, parent[i]);
    createBlockPoolVersionFile(bpDir, version, bpidToWrite);
  }
}
 

import org.apache.hadoop.hdfs.server.datanode.BlockPoolSliceStorage; //导入依赖的package包/类
/**
 * Verify that the current directory exists and that the previous directory
 * does not exist.  Verify that current hasn't been modified by comparing 
 * the checksum of all it's containing files with their original checksum.
 */
static void checkResult(String[] nameNodeDirs, String[] dataNodeDirs,
  String bpid) throws Exception {
  List<File> dirs = Lists.newArrayList();
  for (int i = 0; i < nameNodeDirs.length; i++) {
    File curDir = new File(nameNodeDirs[i], "current");
    dirs.add(curDir);
    FSImageTestUtil.assertReasonableNameCurrentDir(curDir);
  }
  
  FSImageTestUtil.assertParallelFilesAreIdentical(
      dirs, Collections.<String>emptySet());
  
  File dnCurDirs[] = new File[dataNodeDirs.length];
  for (int i = 0; i < dataNodeDirs.length; i++) {
    dnCurDirs[i] = new File(dataNodeDirs[i],"current");
    assertEquals(UpgradeUtilities.checksumContents(DATA_NODE, dnCurDirs[i],
            false), UpgradeUtilities.checksumMasterDataNodeContents());
  }
  for (int i = 0; i < nameNodeDirs.length; i++) {
    assertFalse(new File(nameNodeDirs[i],"previous").isDirectory());
  }

  if (bpid == null) {
    for (int i = 0; i < dataNodeDirs.length; i++) {
      assertFalse(new File(dataNodeDirs[i],"previous").isDirectory());
    }
  } else {
    for (int i = 0; i < dataNodeDirs.length; i++) {
      File bpRoot = BlockPoolSliceStorage.getBpRoot(bpid, dnCurDirs[i]);
      assertFalse(new File(bpRoot,"previous").isDirectory());
      
      File bpCurFinalizeDir = new File(bpRoot,"current/"+DataStorage.STORAGE_DIR_FINALIZED);
      assertEquals(UpgradeUtilities.checksumContents(DATA_NODE,
              bpCurFinalizeDir, true),
              UpgradeUtilities.checksumMasterBlockPoolFinalizedContents());
    }
  }
}
 

import org.apache.hadoop.hdfs.server.datanode.BlockPoolSliceStorage; //导入依赖的package包/类
/** Disable the logs that are not very useful for snapshot related tests. */
public static void disableLogs() {
  final String[] lognames = {
      "org.apache.hadoop.hdfs.server.datanode.BlockPoolSliceScanner",
      "org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl",
      "org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetAsyncDiskService",
  };
  for(String n : lognames) {
    GenericTestUtils.disableLog(LogFactory.getLog(n));
  }
  
  GenericTestUtils.disableLog(LogFactory.getLog(UserGroupInformation.class));
  GenericTestUtils.disableLog(LogFactory.getLog(BlockManager.class));
  GenericTestUtils.disableLog(LogFactory.getLog(FSNamesystem.class));
  GenericTestUtils.disableLog(LogFactory.getLog(DirectoryScanner.class));
  GenericTestUtils.disableLog(LogFactory.getLog(MetricsSystemImpl.class));
  
  GenericTestUtils.disableLog(BlockScanner.LOG);
  GenericTestUtils.disableLog(HttpServer2.LOG);
  GenericTestUtils.disableLog(DataNode.LOG);
  GenericTestUtils.disableLog(BlockPoolSliceStorage.LOG);
  GenericTestUtils.disableLog(LeaseManager.LOG);
  GenericTestUtils.disableLog(NameNode.stateChangeLog);
  GenericTestUtils.disableLog(NameNode.blockStateChangeLog);
  GenericTestUtils.disableLog(DFSClient.LOG);
  GenericTestUtils.disableLog(Server.LOG);
}
 

import org.apache.hadoop.hdfs.server.datanode.BlockPoolSliceStorage; //导入依赖的package包/类
public static void createBlockPoolVersionFile(File bpDir,
    StorageInfo version, String bpid) throws IOException {
  // Create block pool version files
  if (DataNodeLayoutVersion.supports(
      LayoutVersion.Feature.FEDERATION, version.layoutVersion)) {
    File bpCurDir = new File(bpDir, Storage.STORAGE_DIR_CURRENT);
    BlockPoolSliceStorage bpStorage = new BlockPoolSliceStorage(version,
        bpid);
    File versionFile = new File(bpCurDir, "VERSION");
    StorageDirectory sd = new StorageDirectory(bpDir);
    bpStorage.writeProperties(versionFile, sd);
  }
}
 

import org.apache.hadoop.hdfs.server.datanode.BlockPoolSliceStorage; //导入依赖的package包/类
/** Disable the logs that are not very useful for snapshot related tests. */
public static void disableLogs() {
  final String[] lognames = {
      "org.apache.hadoop.hdfs.server.datanode.BlockPoolSliceScanner",
      "org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl",
      "org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetAsyncDiskService",
  };
  for(String n : lognames) {
    setLevel2OFF(LogFactory.getLog(n));
  }
  
  setLevel2OFF(LogFactory.getLog(UserGroupInformation.class));
  setLevel2OFF(LogFactory.getLog(BlockManager.class));
  setLevel2OFF(LogFactory.getLog(FSNamesystem.class));
  setLevel2OFF(LogFactory.getLog(DirectoryScanner.class));
  setLevel2OFF(LogFactory.getLog(MetricsSystemImpl.class));
  
  setLevel2OFF(DataBlockScanner.LOG);
  setLevel2OFF(HttpServer.LOG);
  setLevel2OFF(DataNode.LOG);
  setLevel2OFF(BlockPoolSliceStorage.LOG);
  setLevel2OFF(LeaseManager.LOG);
  setLevel2OFF(NameNode.stateChangeLog);
  setLevel2OFF(NameNode.blockStateChangeLog);
  setLevel2OFF(DFSClient.LOG);
  setLevel2OFF(Server.LOG);
}
 

import org.apache.hadoop.hdfs.server.datanode.BlockPoolSliceStorage; //导入依赖的package包/类
/**
 * Create a <code>version</code> file for datanode inside the specified parent
 * directory.  If such a file already exists, it will be overwritten.
 * The given version string will be written to the file as the layout
 * version. None of the parameters may be null.
 *
 * @param parent directory where namenode VERSION file is stored
 * @param version StorageInfo to create VERSION file from
 * @param bpid Block pool Id
 * @param bpidToWrite Block pool Id to write into the version file
 */
public static void createDataNodeVersionFile(File[] parent,
    StorageInfo version, String bpid, String bpidToWrite) throws IOException {
  DataStorage storage = new DataStorage(version, "doNotCare");

  File[] versionFiles = new File[parent.length];
  for (int i = 0; i < parent.length; i++) {
    File versionFile = new File(parent[i], "VERSION");
    StorageDirectory sd = new StorageDirectory(parent[i].getParentFile());
    storage.writeProperties(versionFile, sd);
    versionFiles[i] = versionFile;
    File bpDir = BlockPoolSliceStorage.getBpRoot(bpid, parent[i]);
    createBlockPoolVersionFile(bpDir, version, bpidToWrite);
  }
}
 

import org.apache.hadoop.hdfs.server.datanode.BlockPoolSliceStorage; //导入依赖的package包/类
public static void createBlockPoolVersionFile(File bpDir,
    StorageInfo version, String bpid) throws IOException {
  // Create block pool version files
  if (LayoutVersion.supports(Feature.FEDERATION, version.layoutVersion)) {
    File bpCurDir = new File(bpDir, Storage.STORAGE_DIR_CURRENT);
    BlockPoolSliceStorage bpStorage = new BlockPoolSliceStorage(version,
        bpid);
    File versionFile = new File(bpCurDir, "VERSION");
    StorageDirectory sd = new StorageDirectory(bpDir);
    bpStorage.writeProperties(versionFile, sd);
  }
}
 

import org.apache.hadoop.hdfs.server.datanode.BlockPoolSliceStorage; //导入依赖的package包/类
/** Disable the logs that are not very useful for snapshot related tests. */
public static void disableLogs() {
  final String[] lognames = {
      "org.apache.hadoop.hdfs.server.datanode.BlockPoolSliceScanner",
      "org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl",
      "org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetAsyncDiskService",
  };
  for(String n : lognames) {
    setLevel2OFF(LogFactory.getLog(n));
  }
  
  setLevel2OFF(LogFactory.getLog(UserGroupInformation.class));
  setLevel2OFF(LogFactory.getLog(BlockManager.class));
  setLevel2OFF(LogFactory.getLog(FSNamesystem.class));
  setLevel2OFF(LogFactory.getLog(DirectoryScanner.class));
  setLevel2OFF(LogFactory.getLog(MetricsSystemImpl.class));
  
  setLevel2OFF(DataBlockScanner.LOG);
  setLevel2OFF(HttpServer2.LOG);
  setLevel2OFF(DataNode.LOG);
  setLevel2OFF(BlockPoolSliceStorage.LOG);
  setLevel2OFF(LeaseManager.LOG);
  setLevel2OFF(NameNode.stateChangeLog);
  setLevel2OFF(NameNode.blockStateChangeLog);
  setLevel2OFF(DFSClient.LOG);
  setLevel2OFF(Server.LOG);
}
 

import org.apache.hadoop.hdfs.server.datanode.BlockPoolSliceStorage; //导入依赖的package包/类
public static void createBlockPoolVersionFile(File bpDir, StorageInfo version,
    String bpid) throws IOException {
  // Create block pool version files
  if (LayoutVersion.supports(Feature.FEDERATION, version.layoutVersion)) {
    File bpCurDir = new File(bpDir, Storage.STORAGE_DIR_CURRENT);
    BlockPoolSliceStorage bpStorage =
        new BlockPoolSliceStorage(version, bpid);
    File versionFile = new File(bpCurDir, "VERSION");
    StorageDirectory sd = new StorageDirectory(bpDir);
    bpStorage.writeProperties(versionFile, sd);
  }
}
 

import org.apache.hadoop.hdfs.server.datanode.BlockPoolSliceStorage; //导入依赖的package包/类
/**
 * Initialize the data structures used by this class.
 * IMPORTANT NOTE: This method must be called once before calling
 * any other public method on this class.
 * <p/>
 * Creates a singleton master populated storage
 * directory for a Namenode (contains edits, fsimage,
 * version, and time files) and a Datanode (contains version and
 * block files).  This can be a lengthy operation.
 */
public static void initialize() throws Exception {
  createEmptyDirs(new String[]{TEST_ROOT_DIR.toString()});
  Configuration config = new HdfsConfiguration();
  config.set(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY,
      datanodeStorage.toString());
  MiniDFSCluster cluster = null;
  try {
    // format data-node
    createEmptyDirs(new String[]{datanodeStorage.toString()});
    
    // format and start NameNode and start DataNode
    DFSTestUtil.formatNameNode(config);
    cluster = new MiniDFSCluster.Builder(config).numDataNodes(1)
        .startupOption(StartupOption.REGULAR).format(false)
        .manageDataDfsDirs(false).manageNameDfsDirs(false).build();

    NamenodeProtocols namenode = cluster.getNameNodeRpc();
    namenodeStorageNamespaceID = namenode.versionRequest().getNamespaceID();
    namenodeStorageFsscTime = namenode.versionRequest().getCTime();
    namenodeStorageClusterID = namenode.versionRequest().getClusterID();
    namenodeStorageBlockPoolID = namenode.versionRequest().getBlockPoolID();
    
    FileSystem fs = FileSystem.get(config);
    Path baseDir = new Path("/TestUpgrade");
    fs.mkdirs(baseDir);
    
    // write some files
    int bufferSize = 4096;
    byte[] buffer = new byte[bufferSize];
    for (int i = 0; i < bufferSize; i++) {
      buffer[i] = (byte) ('0' + i % 50);
    }
    writeFile(fs, new Path(baseDir, "file1"), buffer, bufferSize);
    writeFile(fs, new Path(baseDir, "file2"), buffer, bufferSize);

    
    // write more files
    writeFile(fs, new Path(baseDir, "file3"), buffer, bufferSize);
    writeFile(fs, new Path(baseDir, "file4"), buffer, bufferSize);
  } finally {
    // shutdown
    if (cluster != null) {
      cluster.shutdown();
    }
    FileUtil.fullyDelete(new File(namenodeStorage, "in_use.lock"));
    FileUtil.fullyDelete(new File(datanodeStorage, "in_use.lock"));
  }
  File dnCurDir = new File(datanodeStorage, "current");
  datanodeStorageChecksum = checksumContents(DATA_NODE, dnCurDir);
  
  String bpid = cluster.getNamesystem(0).getBlockPoolId();
  File bpCurDir =
      new File(BlockPoolSliceStorage.getBpRoot(bpid, dnCurDir), "current");
  blockPoolStorageChecksum = checksumContents(DATA_NODE, bpCurDir);
  
  File bpCurFinalizeDir =
      new File(BlockPoolSliceStorage.getBpRoot(bpid, dnCurDir),
          "current/" + DataStorage.STORAGE_DIR_FINALIZED);
  blockPoolFinalizedStorageChecksum =
      checksumContents(DATA_NODE, bpCurFinalizeDir);
  
  File bpCurRbwDir = new File(BlockPoolSliceStorage.getBpRoot(bpid, dnCurDir),
      "current/" + DataStorage.STORAGE_DIR_RBW);
  blockPoolRbwStorageChecksum = checksumContents(DATA_NODE, bpCurRbwDir);
}
 

import org.apache.hadoop.hdfs.server.datanode.BlockPoolSliceStorage; //导入依赖的package包/类
/**
 * Create a <code>version</code> file for datanode inside the specified
 * parent
 * directory.  If such a file already exists, it will be overwritten.
 * The given version string will be written to the file as the layout
 * version. None of the parameters may be null.
 *
 * @param parent
 *     directory where namenode VERSION file is stored
 * @param version
 *     StorageInfo to create VERSION file from
 * @param bpid
 *     Block pool Id
 * @param bpidToWrite
 *     Block pool Id to write into the version file
 */
public static void createDataNodeVersionFile(File[] parent,
    StorageInfo version, String bpid, String bpidToWrite) throws IOException {
  DataStorage storage = new DataStorage(version, "doNotCare");

  File[] versionFiles = new File[parent.length];
  for (int i = 0; i < parent.length; i++) {
    File versionFile = new File(parent[i], "VERSION");
    StorageDirectory sd = new StorageDirectory(parent[i].getParentFile());
    storage.writeProperties(versionFile, sd);
    versionFiles[i] = versionFile;
    File bpDir = BlockPoolSliceStorage.getBpRoot(bpid, parent[i]);
    createBlockPoolVersionFile(bpDir, version, bpidToWrite);
  }
}