Java SnapshotReferenceUtil类代码示例

import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil; //导入依赖的package包/类
/**
 * This method should only be called <b>once</b>, as it starts a thread to keep the cache
 * up-to-date.
 * <p>
 * {@inheritDoc}
 */
@Override
public void setConf(Configuration conf) {
  super.setConf(conf);
  try {
    long cacheRefreshPeriod = conf.getLong(
      WAL_CACHE_REFRESH_PERIOD_CONF_KEY, DEFAULT_WAL_CACHE_REFRESH_PERIOD);
    final FileSystem fs = FSUtils.getCurrentFileSystem(conf);
    Path rootDir = FSUtils.getRootDir(conf);
    cache = new SnapshotFileCache(fs, rootDir, cacheRefreshPeriod, cacheRefreshPeriod,
        "snapshot-log-cleaner-cache-refresher", new SnapshotFileCache.SnapshotFileInspector() {
          public Collection<String> filesUnderSnapshot(final Path snapshotDir)
              throws IOException {
            return SnapshotReferenceUtil.getWALNames(fs, snapshotDir);
          }
        });
  } catch (IOException e) {
    LOG.error("Failed to create snapshot log cleaner", e);
  }
}
 

import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil; //导入依赖的package包/类
public void setConf(final Configuration conf) {
  super.setConf(conf);
  try {
    long cacheRefreshPeriod = conf.getLong(HFILE_CACHE_REFRESH_PERIOD_CONF_KEY,
      DEFAULT_HFILE_CACHE_REFRESH_PERIOD);
    final FileSystem fs = FSUtils.getCurrentFileSystem(conf);
    Path rootDir = FSUtils.getRootDir(conf);
    cache = new SnapshotFileCache(fs, rootDir, cacheRefreshPeriod, cacheRefreshPeriod,
        "snapshot-hfile-cleaner-cache-refresher", new SnapshotFileCache.SnapshotFileInspector() {
          public Collection<String> filesUnderSnapshot(final Path snapshotDir)
              throws IOException {
            return SnapshotReferenceUtil.getHFileNames(conf, fs, snapshotDir);
          }
        });
  } catch (IOException e) {
    LOG.error("Failed to create cleaner util", e);
  }
}
 

import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil; //导入依赖的package包/类
/**
 * This method should only be called <b>once</b>, as it starts a thread to keep the cache
 * up-to-date.
 * <p>
 * {@inheritDoc}
 */
@Override
public void setConf(Configuration conf) {
  super.setConf(conf);
  try {
    long cacheRefreshPeriod = conf.getLong(
      HLOG_CACHE_REFRESH_PERIOD_CONF_KEY, DEFAULT_HLOG_CACHE_REFRESH_PERIOD);
    final FileSystem fs = FSUtils.getCurrentFileSystem(conf);
    Path rootDir = FSUtils.getRootDir(conf);
    cache = new SnapshotFileCache(fs, rootDir, cacheRefreshPeriod, cacheRefreshPeriod,
        "snapshot-log-cleaner-cache-refresher", new SnapshotFileCache.SnapshotFileInspector() {
          public Collection<String> filesUnderSnapshot(final Path snapshotDir)
              throws IOException {
            return SnapshotReferenceUtil.getHLogNames(fs, snapshotDir);
          }
        });
  } catch (IOException e) {
    LOG.error("Failed to create snapshot log cleaner", e);
  }
}
 

import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil; //导入依赖的package包/类
@Override
public void setConf(Configuration conf) {
  super.setConf(conf);
  try {
    long cacheRefreshPeriod = conf.getLong(HFILE_CACHE_REFRESH_PERIOD_CONF_KEY,
      DEFAULT_HFILE_CACHE_REFRESH_PERIOD);
    final FileSystem fs = FSUtils.getCurrentFileSystem(conf);
    Path rootDir = FSUtils.getRootDir(conf);
    cache = new SnapshotFileCache(fs, rootDir, cacheRefreshPeriod, cacheRefreshPeriod,
        "snapshot-hfile-cleaner-cache-refresher", new SnapshotFileCache.SnapshotFileInspector() {
          public Collection<String> filesUnderSnapshot(final Path snapshotDir)
              throws IOException {
            return SnapshotReferenceUtil.getHFileNames(fs, snapshotDir);
          }
        });
  } catch (IOException e) {
    LOG.error("Failed to create cleaner util", e);
  }
}
 

import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil; //导入依赖的package包/类
/**
 * Corrupt the specified snapshot by deleting some files.
 *
 * @param util {@link HBaseTestingUtility}
 * @param snapshotName name of the snapshot to corrupt
 * @return array of the corrupted HFiles
 * @throws IOException on unexecpted error reading the FS
 */
public static ArrayList corruptSnapshot(final HBaseTestingUtility util, final String snapshotName)
    throws IOException {
  final MasterFileSystem mfs = util.getHBaseCluster().getMaster().getMasterFileSystem();
  final FileSystem fs = mfs.getFileSystem();

  Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName,
                                                                      mfs.getRootDir());
  SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
  final String table = snapshotDesc.getTable();

  final ArrayList corruptedFiles = new ArrayList();
  SnapshotReferenceUtil.visitTableStoreFiles(fs, snapshotDir, new FSVisitor.StoreFileVisitor() {
    public void storeFile (final String region, final String family, final String hfile)
        throws IOException {
      HFileLink link = HFileLink.create(util.getConfiguration(), table, region, family, hfile);
      if (corruptedFiles.size() % 2 == 0) {
        fs.delete(link.getAvailablePath(fs));
        corruptedFiles.add(hfile);
      }
    }
  });

  assertTrue(corruptedFiles.size() > 0);
  return corruptedFiles;
}
 

import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil; //导入依赖的package包/类
@Override
public void setConf(final Configuration conf) {
  super.setConf(conf);
  try {
    long cacheRefreshPeriod = conf.getLong(HFILE_CACHE_REFRESH_PERIOD_CONF_KEY,
      DEFAULT_HFILE_CACHE_REFRESH_PERIOD);
    final FileSystem fs = FSUtils.getCurrentFileSystem(conf);
    Path rootDir = FSUtils.getRootDir(conf);
    cache = new SnapshotFileCache(fs, rootDir, cacheRefreshPeriod, cacheRefreshPeriod,
        "snapshot-hfile-cleaner-cache-refresher", new SnapshotFileCache.SnapshotFileInspector() {
          @Override
          public Collection<String> filesUnderSnapshot(final Path snapshotDir)
              throws IOException {
            return SnapshotReferenceUtil.getHFileNames(conf, fs, snapshotDir);
          }
        });
  } catch (IOException e) {
    LOG.error("Failed to create cleaner util", e);
  }
}
 

import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil; //导入依赖的package包/类
@Override
public void setConf(final Configuration conf) {
  super.setConf(conf);
  try {
    long cacheRefreshPeriod = conf.getLong(HFILE_CACHE_REFRESH_PERIOD_CONF_KEY,
      DEFAULT_HFILE_CACHE_REFRESH_PERIOD);
    final FileSystem fs = FSUtils.getCurrentFileSystem(conf);
    Path rootDir = FSUtils.getRootDir(conf);
    cache = new SnapshotFileCache(fs, rootDir, cacheRefreshPeriod, cacheRefreshPeriod,
        "snapshot-hfile-cleaner-cache-refresher", new SnapshotFileCache.SnapshotFileInspector() {
          public Collection<String> filesUnderSnapshot(final Path snapshotDir)
              throws IOException {
            return SnapshotReferenceUtil.getHFileNames(conf, fs, snapshotDir);
          }
        });
  } catch (IOException e) {
    LOG.error("Failed to create cleaner util", e);
  }
}
 

import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil; //导入依赖的package包/类
/**
 * Verify that the region (regioninfo, hfiles) are valid
 * @param fs the FileSystem instance
 * @param snapshotDir snapshot directory to check
 * @param region the region to check
 */
private void verifyRegion(final FileSystem fs, final Path snapshotDir, final HRegionInfo region)
    throws IOException {
  // make sure we have region in the snapshot
  Path regionDir = new Path(snapshotDir, region.getEncodedName());

  // make sure we have the region info in the snapshot
  Path regionInfo = new Path(regionDir, HRegion.REGIONINFO_FILE);
  // make sure the file exists
  if (!fs.exists(regionInfo)) {
    throw new CorruptedSnapshotException("No region info found for region:" + region, snapshot);
  }

  FSDataInputStream in = fs.open(regionInfo);
  HRegionInfo found = new HRegionInfo();
  try {
    found.readFields(in);
    if (!region.equals(found)) {
      throw new CorruptedSnapshotException("Found region info (" + found
         + ") doesn't match expected region:" + region, snapshot);
    }
  } finally {
    in.close();
  }

  // make sure we have the expected recovered edits files
  TakeSnapshotUtils.verifyRecoveredEdits(fs, snapshotDir, found, snapshot);

  // make sure we have all the expected store files
  SnapshotReferenceUtil.visitRegionStoreFiles(fs, regionDir, new FSVisitor.StoreFileVisitor() {
    public void storeFile(final String regionNameSuffix, final String family,
        final String hfileName) throws IOException {
      verifyStoreFile(snapshotDir, region, family, hfileName);
    }
  });
}
 

import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil; //导入依赖的package包/类
/**
 * Add the specified recovered.edits file to the stats
 * @param region region encoded name
 * @param logfile log file name
 * @return the recovered.edits information
 */
FileInfo addRecoveredEdits(final String region, final String logfile) throws IOException {
  Path rootDir = FSUtils.getRootDir(conf);
  Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir);
  Path path = SnapshotReferenceUtil.getRecoveredEdits(snapshotDir, region, logfile);
  long size = fs.getFileStatus(path).getLen();
  logSize += size;
  logsCount++;
  return new FileInfo(true, size);
}
 

import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil; //导入依赖的package包/类
private void init() throws IOException {
  Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);

  //load table descriptor
  htd = FSTableDescriptors.getTableDescriptorFromFs(fs, snapshotDir);

  Set<String> snapshotRegionNames
    = SnapshotReferenceUtil.getSnapshotRegionNames(fs, snapshotDir);
  if (snapshotRegionNames == null) {
    throw new IllegalArgumentException("Snapshot seems empty");
  }

  regions = new ArrayList<HRegionInfo>(snapshotRegionNames.size());
  for (String regionName : snapshotRegionNames) {
    // load region descriptor
    Path regionDir = new Path(snapshotDir, regionName);
    HRegionInfo hri = HRegionFileSystem.loadRegionInfoFileContent(fs,
        regionDir);

    if (CellUtil.overlappingKeys(scan.getStartRow(), scan.getStopRow(),
        hri.getStartKey(), hri.getEndKey())) {
      regions.add(hri);
    }
  }

  // sort for regions according to startKey.
  Collections.sort(regions);

  initScanMetrics(scan);

  RestoreSnapshotHelper.copySnapshotForScanner(conf, fs,
    rootDir, restoreDir, snapshotName);
}
 

import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil; //导入依赖的package包/类
/**
 * Check that all the regions in the snapshot are valid, and accounted for.
 * @param snapshotDir snapshot directory to check
 * @throws IOException if we can't reach hbase:meta or read the files from the FS
 */
private void verifyRegions(Path snapshotDir) throws IOException {
  List<HRegionInfo> regions = MetaReader.getTableRegions(this.services.getCatalogTracker(),
      tableName);

  Set<String> snapshotRegions = SnapshotReferenceUtil.getSnapshotRegionNames(fs, snapshotDir);
  if (snapshotRegions == null) {
    String msg = "Snapshot " + ClientSnapshotDescriptionUtils.toString(snapshot) + " looks empty";
    LOG.error(msg);
    throw new CorruptedSnapshotException(msg);
  }

  String errorMsg = "";
  if (snapshotRegions.size() != regions.size()) {
    errorMsg = "Regions moved during the snapshot '" + 
                 ClientSnapshotDescriptionUtils.toString(snapshot) + "'. expected=" +
                 regions.size() + " snapshotted=" + snapshotRegions.size() + ".";
    LOG.error(errorMsg);
  }

  for (HRegionInfo region : regions) {
    if (!snapshotRegions.contains(region.getEncodedName())) {
      // could happen due to a move or split race.
      String mesg = " No snapshot region directory found for region:" + region;
      if (errorMsg.isEmpty()) errorMsg = mesg;
      LOG.error(mesg);
    }

    verifyRegion(fs, snapshotDir, region);
  }
  if (!errorMsg.isEmpty()) {
    throw new CorruptedSnapshotException(errorMsg);
  }
}
 

import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil; //导入依赖的package包/类
/**
 * Verify that the region (regioninfo, hfiles) are valid
 * @param fs the FileSystem instance
 * @param snapshotDir snapshot directory to check
 * @param region the region to check
 */
private void verifyRegion(final FileSystem fs, final Path snapshotDir, final HRegionInfo region)
    throws IOException {
  // make sure we have region in the snapshot
  Path regionDir = new Path(snapshotDir, region.getEncodedName());

  // make sure we have the region info in the snapshot
  Path regionInfo = new Path(regionDir, HRegionFileSystem.REGION_INFO_FILE);
  // make sure the file exists
  if (!fs.exists(regionInfo)) {
    throw new CorruptedSnapshotException("No region info found for region:" + region, snapshot);
  }

  HRegionInfo found = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir);
  if (!region.equals(found)) {
    throw new CorruptedSnapshotException("Found region info (" + found
      + ") doesn't match expected region:" + region, snapshot);
  }

  // make sure we have the expected recovered edits files
  TakeSnapshotUtils.verifyRecoveredEdits(fs, snapshotDir, found, snapshot);

   // make sure we have all the expected store files
  SnapshotReferenceUtil.visitRegionStoreFiles(fs, regionDir, new FSVisitor.StoreFileVisitor() {
    public void storeFile(final String regionNameSuffix, final String family,
        final String hfileName) throws IOException {
      verifyStoreFile(snapshotDir, region, family, hfileName);
    }
  });
}
 

import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil; //导入依赖的package包/类
/**
 * Corrupt the specified snapshot by deleting some files.
 *
 * @param util {@link HBaseTestingUtility}
 * @param snapshotName name of the snapshot to corrupt
 * @return array of the corrupted HFiles
 * @throws IOException on unexecpted error reading the FS
 */
public static ArrayList corruptSnapshot(final HBaseTestingUtility util, final String snapshotName)
    throws IOException {
  final MasterFileSystem mfs = util.getHBaseCluster().getMaster().getMasterFileSystem();
  final FileSystem fs = mfs.getFileSystem();

  Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName,
                                                                      mfs.getRootDir());
  SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
  final TableName table = TableName.valueOf(snapshotDesc.getTable());

  final ArrayList corruptedFiles = new ArrayList();
  SnapshotReferenceUtil.visitTableStoreFiles(fs, snapshotDir, new FSVisitor.StoreFileVisitor() {
    @Override
    public void storeFile (final String region, final String family, final String hfile)
        throws IOException {
      HFileLink link = HFileLink.create(util.getConfiguration(), table, region, family, hfile);
      if (corruptedFiles.size() % 2 == 0) {
        fs.delete(link.getAvailablePath(fs));
        corruptedFiles.add(hfile);
      }
    }
  });

  assertTrue(corruptedFiles.size() > 0);
  return corruptedFiles;
}
 

import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil; //导入依赖的package包/类
/**
 * Restore or Clone the specified snapshot
 * @param reqSnapshot
 * @param nonceKey unique identifier to prevent duplicated RPC
 * @throws IOException
 */
public long restoreOrCloneSnapshot(final SnapshotDescription reqSnapshot, final NonceKey nonceKey,
    final boolean restoreAcl) throws IOException {
  FileSystem fs = master.getMasterFileSystem().getFileSystem();
  Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(reqSnapshot, rootDir);

  // check if the snapshot exists
  if (!fs.exists(snapshotDir)) {
    LOG.error("A Snapshot named '" + reqSnapshot.getName() + "' does not exist.");
    throw new SnapshotDoesNotExistException(
      ProtobufUtil.createSnapshotDesc(reqSnapshot));
  }

  // Get snapshot info from file system. The reqSnapshot is a "fake" snapshotInfo with
  // just the snapshot "name" and table name to restore. It does not contains the "real" snapshot
  // information.
  SnapshotDescription snapshot = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
  SnapshotManifest manifest = SnapshotManifest.open(master.getConfiguration(), fs,
      snapshotDir, snapshot);
  TableDescriptor snapshotTableDesc = manifest.getTableDescriptor();
  TableName tableName = TableName.valueOf(reqSnapshot.getTable());

  // stop tracking "abandoned" handlers
  cleanupSentinels();

  // Verify snapshot validity
  SnapshotReferenceUtil.verifySnapshot(master.getConfiguration(), fs, manifest);

  // Execute the restore/clone operation
  long procId;
  if (MetaTableAccessor.tableExists(master.getConnection(), tableName)) {
    procId = restoreSnapshot(reqSnapshot, tableName, snapshot, snapshotTableDesc, nonceKey,
      restoreAcl);
  } else {
    procId =
        cloneSnapshot(reqSnapshot, tableName, snapshot, snapshotTableDesc, nonceKey, restoreAcl);
  }
  return procId;
}
 

import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil; //导入依赖的package包/类
/**
 * Check that all the regions in the snapshot are valid, and accounted for.
 * @param manifest snapshot manifest to inspect
 * @throws IOException if we can't reach hbase:meta or read the files from the FS
 */
private void verifyRegions(final SnapshotManifest manifest) throws IOException {
  List<HRegionInfo> regions;
  if (TableName.META_TABLE_NAME.equals(tableName)) {
    regions = new MetaTableLocator().getMetaRegions(services.getZooKeeper());
  } else {
    regions = MetaTableAccessor.getTableRegions(services.getZooKeeper(),
      services.getConnection(), tableName);
  }
  // Remove the non-default regions
  RegionReplicaUtil.removeNonDefaultRegions(regions);

  Map<String, SnapshotRegionManifest> regionManifests = manifest.getRegionManifestsMap();
  if (regionManifests == null) {
    String msg = "Snapshot " + ClientSnapshotDescriptionUtils.toString(snapshot) + " looks empty";
    LOG.error(msg);
    throw new CorruptedSnapshotException(msg);
  }

  String errorMsg = "";
  if (regionManifests.size() != regions.size()) {
    errorMsg = "Regions moved during the snapshot '" +
                 ClientSnapshotDescriptionUtils.toString(snapshot) + "'. expected=" +
                 regions.size() + " snapshotted=" + regionManifests.size() + ".";
    LOG.error(errorMsg);
  }

  // Verify HRegionInfo
  for (HRegionInfo region : regions) {
    SnapshotRegionManifest regionManifest = regionManifests.get(region.getEncodedName());
    if (regionManifest == null) {
      // could happen due to a move or split race.
      String mesg = " No snapshot region directory found for region:" + region;
      if (errorMsg.isEmpty()) errorMsg = mesg;
      LOG.error(mesg);
      continue;
    }

    verifyRegionInfo(region, regionManifest);
  }

  if (!errorMsg.isEmpty()) {
    throw new CorruptedSnapshotException(errorMsg);
  }

  // Verify Snapshot HFiles
  SnapshotReferenceUtil.verifySnapshot(services.getConfiguration(), fs, manifest);
}