Java HFileLinkCleaner类代码示例

import org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner; //导入依赖的package包/类
/**
 * Called at startup, to verify if snapshot operation is supported, and to avoid
 * starting the master if there're snapshots present but the cleaners needed are missing.
 * Otherwise we can end up with snapshot data loss.
 * @param conf The {@link Configuration} object to use
 * @param mfs The MasterFileSystem to use
 * @throws IOException in case of file-system operation failure
 * @throws UnsupportedOperationException in case cleaners are missing and
 *         there're snapshot in the system
 */
private void checkSnapshotSupport(final Configuration conf, final MasterFileSystem mfs)
    throws IOException, UnsupportedOperationException {
  // Verify if snapshot is disabled by the user
  String enabled = conf.get(HBASE_SNAPSHOT_ENABLED);
  boolean snapshotEnabled = conf.getBoolean(HBASE_SNAPSHOT_ENABLED, false);
  boolean userDisabled = (enabled != null && enabled.trim().length() > 0 && !snapshotEnabled);

  // Extract cleaners from conf
  Set<String> hfileCleaners = new HashSet<String>();
  String[] cleaners = conf.getStrings(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS);
  if (cleaners != null) Collections.addAll(hfileCleaners, cleaners);

  Set<String> logCleaners = new HashSet<String>();
  cleaners = conf.getStrings(HConstants.HBASE_MASTER_LOGCLEANER_PLUGINS);
  if (cleaners != null) Collections.addAll(logCleaners, cleaners);

  // check if an older version of snapshot directory was present
  Path oldSnapshotDir = new Path(mfs.getRootDir(), HConstants.OLD_SNAPSHOT_DIR_NAME);
  FileSystem fs = mfs.getFileSystem();
  List<SnapshotDescription> ss = getCompletedSnapshots(new Path(rootDir, oldSnapshotDir));
  if (ss != null && !ss.isEmpty()) {
    LOG.error("Snapshots from an earlier release were found under: " + oldSnapshotDir);
    LOG.error("Please rename the directory as " + HConstants.SNAPSHOT_DIR_NAME);
  }

  // If the user has enabled the snapshot, we force the cleaners to be present
  // otherwise we still need to check if cleaners are enabled or not and verify
  // that there're no snapshot in the .snapshot folder.
  if (snapshotEnabled) {
    // Inject snapshot cleaners, if snapshot.enable is true
    hfileCleaners.add(SnapshotHFileCleaner.class.getName());
    hfileCleaners.add(HFileLinkCleaner.class.getName());
    logCleaners.add(SnapshotLogCleaner.class.getName());

    // Set cleaners conf
    conf.setStrings(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS,
      hfileCleaners.toArray(new String[hfileCleaners.size()]));
    conf.setStrings(HConstants.HBASE_MASTER_LOGCLEANER_PLUGINS,
      logCleaners.toArray(new String[logCleaners.size()]));
  } else {
    // Verify if cleaners are present
    snapshotEnabled = logCleaners.contains(SnapshotLogCleaner.class.getName()) &&
      hfileCleaners.contains(SnapshotHFileCleaner.class.getName()) &&
      hfileCleaners.contains(HFileLinkCleaner.class.getName());

    // Warn if the cleaners are enabled but the snapshot.enabled property is false/not set.
    if (snapshotEnabled) {
      LOG.warn("Snapshot log and hfile cleaners are present in the configuration, " +
        "but the '" + HBASE_SNAPSHOT_ENABLED + "' property " +
        (userDisabled ? "is set to 'false'." : "is not set."));
    }
  }

  // Mark snapshot feature as enabled if cleaners are present and user has not disabled it.
  this.isSnapshotSupported = snapshotEnabled && !userDisabled;

  // If cleaners are not enabled, verify that there're no snapshot in the .snapshot folder
  // otherwise we end up with snapshot data loss.
  if (!snapshotEnabled) {
    LOG.info("Snapshot feature is not enabled, missing log and hfile cleaners.");
    Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(mfs.getRootDir());
    if (fs.exists(snapshotDir)) {
      FileStatus[] snapshots = FSUtils.listStatus(fs, snapshotDir,
        new SnapshotDescriptionUtils.CompletedSnaphotDirectoriesFilter(fs));
      if (snapshots != null) {
        LOG.error("Snapshots are present, but cleaners are not enabled.");
        checkSnapshotSupport();
      }
    }
  }
}
 

import org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner; //导入依赖的package包/类
/**
 * Verify the snapshot support based on the configuration.
 */
@Test
public void testSnapshotSupportConfiguration() throws Exception {
  // No configuration (no cleaners, not enabled): snapshot feature disabled
  Configuration conf = new Configuration();
  SnapshotManager manager = getNewManager(conf);
  assertFalse("Snapshot should be disabled with no configuration", isSnapshotSupported(manager));

  // force snapshot feature to be enabled
  conf = new Configuration();
  conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
  manager = getNewManager(conf);
  assertTrue("Snapshot should be enabled", isSnapshotSupported(manager));

  // force snapshot feature to be disabled
  conf = new Configuration();
  conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, false);
  manager = getNewManager(conf);
  assertFalse("Snapshot should be disabled", isSnapshotSupported(manager));

  // force snapshot feature to be disabled, even if cleaners are present
  conf = new Configuration();
  conf.setStrings(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS,
    SnapshotHFileCleaner.class.getName(), HFileLinkCleaner.class.getName());
  conf.set(HConstants.HBASE_MASTER_LOGCLEANER_PLUGINS, SnapshotLogCleaner.class.getName());
  conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, false);
  manager = getNewManager(conf);
  assertFalse("Snapshot should be disabled", isSnapshotSupported(manager));

  // cleaners are present, but missing snapshot enabled property
  conf = new Configuration();
  conf.setStrings(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS,
    SnapshotHFileCleaner.class.getName(), HFileLinkCleaner.class.getName());
  conf.set(HConstants.HBASE_MASTER_LOGCLEANER_PLUGINS, SnapshotLogCleaner.class.getName());
  manager = getNewManager(conf);
  assertTrue("Snapshot should be enabled, because cleaners are present",
    isSnapshotSupported(manager));

  // Create a "test snapshot"
  Path rootDir = UTIL.getDataTestDir();
  Path testSnapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(
    "testSnapshotSupportConfiguration", rootDir);
  fs.mkdirs(testSnapshotDir);
  try {
    // force snapshot feature to be disabled, but snapshots are present
    conf = new Configuration();
    conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, false);
    manager = getNewManager(conf);
    fail("Master should not start when snapshot is disabled, but snapshots are present");
  } catch (UnsupportedOperationException e) {
    // expected
  } finally {
    fs.delete(testSnapshotDir, true);
  }
}
 

import org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner; //导入依赖的package包/类
/**
 * Called at startup, to verify if snapshot operation is supported, and to avoid
 * starting the master if there're snapshots present but the cleaners needed are missing.
 * Otherwise we can end up with snapshot data loss.
 * @param conf The {@link Configuration} object to use
 * @param mfs The MasterFileSystem to use
 * @throws IOException in case of file-system operation failure
 * @throws UnsupportedOperationException in case cleaners are missing and
 *         there're snapshot in the system
 */
private void checkSnapshotSupport(final Configuration conf, final MasterFileSystem mfs)
    throws IOException, UnsupportedOperationException {
  // Verify if snapshot is disabled by the user
  String enabled = conf.get(HBASE_SNAPSHOT_ENABLED);
  boolean snapshotEnabled = conf.getBoolean(HBASE_SNAPSHOT_ENABLED, false);
  boolean userDisabled = (enabled != null && enabled.trim().length() > 0 && !snapshotEnabled);

  // Extract cleaners from conf
  Set<String> hfileCleaners = new HashSet<>();
  String[] cleaners = conf.getStrings(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS);
  if (cleaners != null) Collections.addAll(hfileCleaners, cleaners);

  Set<String> logCleaners = new HashSet<>();
  cleaners = conf.getStrings(HConstants.HBASE_MASTER_LOGCLEANER_PLUGINS);
  if (cleaners != null) Collections.addAll(logCleaners, cleaners);

  // check if an older version of snapshot directory was present
  Path oldSnapshotDir = new Path(mfs.getRootDir(), HConstants.OLD_SNAPSHOT_DIR_NAME);
  FileSystem fs = mfs.getFileSystem();
  List<SnapshotDescription> ss = getCompletedSnapshots(new Path(rootDir, oldSnapshotDir), false);
  if (ss != null && !ss.isEmpty()) {
    LOG.error("Snapshots from an earlier release were found under: " + oldSnapshotDir);
    LOG.error("Please rename the directory as " + HConstants.SNAPSHOT_DIR_NAME);
  }

  // If the user has enabled the snapshot, we force the cleaners to be present
  // otherwise we still need to check if cleaners are enabled or not and verify
  // that there're no snapshot in the .snapshot folder.
  if (snapshotEnabled) {
    // Inject snapshot cleaners, if snapshot.enable is true
    hfileCleaners.add(SnapshotHFileCleaner.class.getName());
    hfileCleaners.add(HFileLinkCleaner.class.getName());

    // Set cleaners conf
    conf.setStrings(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS,
      hfileCleaners.toArray(new String[hfileCleaners.size()]));
    conf.setStrings(HConstants.HBASE_MASTER_LOGCLEANER_PLUGINS,
      logCleaners.toArray(new String[logCleaners.size()]));
  } else {
    // Verify if cleaners are present
    snapshotEnabled =
      hfileCleaners.contains(SnapshotHFileCleaner.class.getName()) &&
      hfileCleaners.contains(HFileLinkCleaner.class.getName());

    // Warn if the cleaners are enabled but the snapshot.enabled property is false/not set.
    if (snapshotEnabled) {
      LOG.warn("Snapshot log and hfile cleaners are present in the configuration, " +
        "but the '" + HBASE_SNAPSHOT_ENABLED + "' property " +
        (userDisabled ? "is set to 'false'." : "is not set."));
    }
  }

  // Mark snapshot feature as enabled if cleaners are present and user has not disabled it.
  this.isSnapshotSupported = snapshotEnabled && !userDisabled;

  // If cleaners are not enabled, verify that there're no snapshot in the .snapshot folder
  // otherwise we end up with snapshot data loss.
  if (!snapshotEnabled) {
    LOG.info("Snapshot feature is not enabled, missing log and hfile cleaners.");
    Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(mfs.getRootDir());
    if (fs.exists(snapshotDir)) {
      FileStatus[] snapshots = FSUtils.listStatus(fs, snapshotDir,
        new SnapshotDescriptionUtils.CompletedSnaphotDirectoriesFilter(fs));
      if (snapshots != null) {
        LOG.error("Snapshots are present, but cleaners are not enabled.");
        checkSnapshotSupport();
      }
    }
  }
}
 

import org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner; //导入依赖的package包/类
/**
 * Verify the snapshot support based on the configuration.
 */
@Test
public void testSnapshotSupportConfiguration() throws Exception {
  // No configuration (no cleaners, not enabled): snapshot feature disabled
  Configuration conf = new Configuration();
  SnapshotManager manager = getNewManager(conf);
  assertFalse("Snapshot should be disabled with no configuration", isSnapshotSupported(manager));

  // force snapshot feature to be enabled
  conf = new Configuration();
  conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
  manager = getNewManager(conf);
  assertTrue("Snapshot should be enabled", isSnapshotSupported(manager));

  // force snapshot feature to be disabled
  conf = new Configuration();
  conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, false);
  manager = getNewManager(conf);
  assertFalse("Snapshot should be disabled", isSnapshotSupported(manager));

  // force snapshot feature to be disabled, even if cleaners are present
  conf = new Configuration();
  conf.setStrings(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS,
    SnapshotHFileCleaner.class.getName(), HFileLinkCleaner.class.getName());
  conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, false);
  manager = getNewManager(conf);
  assertFalse("Snapshot should be disabled", isSnapshotSupported(manager));

  // cleaners are present, but missing snapshot enabled property
  conf = new Configuration();
  conf.setStrings(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS,
    SnapshotHFileCleaner.class.getName(), HFileLinkCleaner.class.getName());
  manager = getNewManager(conf);
  assertTrue("Snapshot should be enabled, because cleaners are present",
    isSnapshotSupported(manager));

  // Create a "test snapshot"
  Path rootDir = UTIL.getDataTestDir();
  Path testSnapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(
    "testSnapshotSupportConfiguration", rootDir);
  fs.mkdirs(testSnapshotDir);
  try {
    // force snapshot feature to be disabled, but snapshots are present
    conf = new Configuration();
    conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, false);
    manager = getNewManager(conf);
    fail("Master should not start when snapshot is disabled, but snapshots are present");
  } catch (UnsupportedOperationException e) {
    // expected
  } finally {
    fs.delete(testSnapshotDir, true);
  }
}