Java HBaseFileSystem.createNewFileOnFileSystem方法代码示例

import org.apache.hadoop.hbase.HBaseFileSystem; //导入方法依赖的package包/类
public static void markCorrupted(Path rootdir, String logFileName,
    FileSystem fs) {
  Path file = new Path(getSplitLogDir(rootdir, logFileName), "corrupt");
  try {
    HBaseFileSystem.createNewFileOnFileSystem(fs, file);
  } catch (IOException e) {
    LOG.warn("Could not flag a log file as corrupted. Failed to create " +
        file, e);
  }
}
 

import org.apache.hadoop.hbase.HBaseFileSystem; //导入方法依赖的package包/类
/**
 * Complete taking the snapshot on the region. Writes the region info and adds references to the
 * working snapshot directory. TODO for api consistency, consider adding another version with no
 * {@link ForeignExceptionSnare} arg. (In the future other cancellable HRegion methods could
 * eventually add a {@link ForeignExceptionSnare}, or we could do something fancier).
 * @param desc snasphot description object
 * @param exnSnare ForeignExceptionSnare that captures external exeptions in case we need to bail
 *          out. This is allowed to be null and will just be ignored in that case.
 * @throws IOException if there is an external or internal error causing the snapshot to fail
 */
public void addRegionToSnapshot(SnapshotDescription desc, ForeignExceptionSnare exnSnare)
    throws IOException {
  // This should be "fast" since we don't rewrite store files but instead
  // back up the store files by creating a reference
  Path rootDir = FSUtils.getRootDir(this.rsServices.getConfiguration());
  Path snapshotRegionDir =
      TakeSnapshotUtils.getRegionSnapshotDirectory(desc, rootDir, regionInfo.getEncodedName());

  // 1. dump region meta info into the snapshot directory
  LOG.debug("Storing region-info for snapshot.");
  checkRegioninfoOnFilesystem(snapshotRegionDir);

  // 2. iterate through all the stores in the region
  LOG.debug("Creating references for hfiles");

  // This ensures that we have an atomic view of the directory as long as we have < ls limit
  // (batch size of the files in a directory) on the namenode. Otherwise, we get back the files in
  // batches and may miss files being added/deleted. This could be more robust (iteratively
  // checking to see if we have all the files until we are sure), but the limit is currently 1000
  // files/batch, far more than the number of store files under a single column family.
  for (Store store : stores.values()) {
    // 2.1. build the snapshot reference directory for the store
    Path dstStoreDir =
        TakeSnapshotUtils.getStoreSnapshotDirectory(snapshotRegionDir,
          Bytes.toString(store.getFamily().getName()));
    List<StoreFile> storeFiles = store.getStorefiles();
    if (LOG.isDebugEnabled()) {
      LOG.debug("Adding snapshot references for " + storeFiles + " hfiles");
    }

    // 2.2. iterate through all the store's files and create "references".
    int sz = storeFiles.size();
    for (int i = 0; i < sz; i++) {
      if (exnSnare != null) {
        exnSnare.rethrowException();
      }
      StoreFile storeFile = storeFiles.get(i);
      Path file = storeFile.getPath();

      LOG.debug("Creating reference for file (" + (i + 1) + "/" + sz + ") : " + file);
      Path referenceFile = new Path(dstStoreDir, file.getName());
      boolean success = true;
      if (storeFile.isReference()) {
        // write the Reference object to the snapshot
        storeFile.getReference().write(fs, referenceFile);
      } else {
        // create "reference" to this store file. It is intentionally an empty file -- all
        // necessary information is captured by its fs location and filename. This allows us to
        // only figure out what needs to be done via a single nn operation (instead of having to
        // open and read the files as well).
        success = HBaseFileSystem.createNewFileOnFileSystem(fs, referenceFile);
      }
      if (!success) {
        throw new IOException("Failed to create reference file:" + referenceFile);
      }
    }
  }
}
 

import org.apache.hadoop.hbase.HBaseFileSystem; //导入方法依赖的package包/类
/**
 * Complete taking the snapshot on the region. Writes the region info and adds references to the
 * working snapshot directory.
 *
 * TODO for api consistency, consider adding another version with no {@link ForeignExceptionSnare}
 * arg.  (In the future other cancellable HRegion methods could eventually add a
 * {@link ForeignExceptionSnare}, or we could do something fancier).
 *
 * @param desc snasphot description object
 * @param exnSnare ForeignExceptionSnare that captures external exeptions in case we need to
 *   bail out.  This is allowed to be null and will just be ignored in that case.
 * @throws IOException if there is an external or internal error causing the snapshot to fail
 */
public void addRegionToSnapshot(SnapshotDescription desc,
    ForeignExceptionSnare exnSnare) throws IOException {
  // This should be "fast" since we don't rewrite store files but instead
  // back up the store files by creating a reference
  Path rootDir = FSUtils.getRootDir(this.rsServices.getConfiguration());
  Path snapshotRegionDir = TakeSnapshotUtils.getRegionSnapshotDirectory(desc, rootDir,
    regionInfo.getEncodedName());

  // 1. dump region meta info into the snapshot directory
  LOG.debug("Storing region-info for snapshot.");
  checkRegioninfoOnFilesystem(snapshotRegionDir);

  // 2. iterate through all the stores in the region
  LOG.debug("Creating references for hfiles");

  // This ensures that we have an atomic view of the directory as long as we have < ls limit
  // (batch size of the files in a directory) on the namenode. Otherwise, we get back the files in
  // batches and may miss files being added/deleted. This could be more robust (iteratively
  // checking to see if we have all the files until we are sure), but the limit is currently 1000
  // files/batch, far more than the number of store files under a single column family.
  for (Store store : stores.values()) {
    // 2.1. build the snapshot reference directory for the store
    Path dstStoreDir = TakeSnapshotUtils.getStoreSnapshotDirectory(snapshotRegionDir,
      Bytes.toString(store.getFamily().getName()));
    List<StoreFile> storeFiles = store.getStorefiles();
    if (LOG.isDebugEnabled()) {
      LOG.debug("Adding snapshot references for " + storeFiles  + " hfiles");
    }

    // 2.2. iterate through all the store's files and create "references".
    int sz = storeFiles.size();
    for (int i = 0; i < sz; i++) {
      if (exnSnare != null) {
        exnSnare.rethrowException();
      }
      StoreFile storeFile = storeFiles.get(i);
      Path file = storeFile.getPath();

      LOG.debug("Creating reference for file (" + (i+1) + "/" + sz + ") : " + file);
      Path referenceFile = new Path(dstStoreDir, file.getName());
      boolean success = true;
      if (storeFile.isReference()) {
        // write the Reference object to the snapshot
        storeFile.getReference().write(fs, referenceFile);
      } else {
        // create "reference" to this store file.  It is intentionally an empty file -- all
        // necessary information is captured by its fs location and filename.  This allows us to
        // only figure out what needs to be done via a single nn operation (instead of having to
        // open and read the files as well).
        success = HBaseFileSystem.createNewFileOnFileSystem(fs, referenceFile);
      }
      if (!success) {
        throw new IOException("Failed to create reference file:" + referenceFile);
      }
    }
  }
}
 

import org.apache.hadoop.hbase.HBaseFileSystem; //导入方法依赖的package包/类
/**
 * Complete taking the snapshot on the region. Writes the region info and adds references to the
 * working snapshot directory.
 *
 * TODO for api consistency, consider adding another version with no {@link ForeignExceptionSnare}
 * arg.  (In the future other cancellable HRegion methods could eventually add a
 * {@link ForeignExceptionSnare}, or we could do something fancier).
 *
 * @param desc snasphot description object
 * @param exnSnare ForeignExceptionSnare that captures external exeptions in case we need to
 *   bail out.  This is allowed to be null and will just be ignored in that case.
 * @throws IOException if there is an external or internal error causing the snapshot to fail
 */
public void addRegionToSnapshot(SnapshotDescription desc,
    ForeignExceptionSnare exnSnare) throws IOException {
  // This should be "fast" since we don't rewrite store files but instead
  // back up the store files by creating a reference
  Path rootDir = FSUtils.getRootDir(this.rsServices.getConfiguration());
  Path snapshotRegionDir = TakeSnapshotUtils.getRegionSnapshotDirectory(desc, rootDir,
    regionInfo.getEncodedName());

  // 1. dump region meta info into the snapshot directory
  LOG.debug("Storing region-info for snapshot.");
  checkRegioninfoOnFilesystem(snapshotRegionDir);

  // 2. iterate through all the stores in the region
  LOG.debug("Creating references for hfiles");

  // This ensures that we have an atomic view of the directory as long as we have < ls limit
  // (batch size of the files in a directory) on the namenode. Otherwise, we get back the files in
  // batches and may miss files being added/deleted. This could be more robust (iteratively
  // checking to see if we have all the files until we are sure), but the limit is currently 1000
  // files/batch, far more than the number of store files under a single column family.
  for (Store store : stores.values()) {
    // 2.1. build the snapshot reference directory for the store
    Path dstStoreDir = TakeSnapshotUtils.getStoreSnapshotDirectory(snapshotRegionDir,
      Bytes.toString(store.getFamily().getName()));
    List<StoreFile> storeFiles = store.getStorefiles();
    if (LOG.isDebugEnabled()) {
      LOG.debug("Adding snapshot references for " + storeFiles  + " hfiles");
    }

    // 2.2. iterate through all the store's files and create "references".
    int sz = storeFiles.size();
    for (int i = 0; i < sz; i++) {
      if (exnSnare != null) {
        exnSnare.rethrowException();
      }
      Path file = storeFiles.get(i).getPath();
      // create "reference" to this store file.  It is intentionally an empty file -- all
      // necessary infomration is captured by its fs location and filename.  This allows us to
      // only figure out what needs to be done via a single nn operation (instead of having to
      // open and read the files as well).
      LOG.debug("Creating reference for file (" + (i+1) + "/" + sz + ") : " + file);
      Path referenceFile = new Path(dstStoreDir, file.getName());
      boolean success = HBaseFileSystem.createNewFileOnFileSystem(fs, referenceFile);
      if (!success) {
        throw new IOException("Failed to create reference file:" + referenceFile);
      }
    }
  }
}