Java HFileLink.isHFileLink方法代码示例

import org.apache.hadoop.hbase.io.HFileLink; //导入方法依赖的package包/类
/**
 * Return if the specified file is a valid store file or not.
 *
 * @param fileStatus The {@link FileStatus} of the file
 * @return <tt>true</tt> if the file is valid
 */
public static boolean isValid(final FileStatus fileStatus) throws IOException {
  final Path p = fileStatus.getPath();

  if (fileStatus.isDirectory()) return false;

  // Check for empty hfile. Should never be the case but can happen
  // after data loss in hdfs for whatever reason (upgrade, etc.): HBASE-646
  // NOTE: that the HFileLink is just a name, so it's an empty file.
  if (!HFileLink.isHFileLink(p) && fileStatus.getLen() <= 0) {
    LOG.warn("Skipping " + p + " because it is empty. HBASE-646 DATA LOSS?");
    return false;
  }

  return validateStoreFileName(p.getName());
}
 

import org.apache.hadoop.hbase.io.HFileLink; //导入方法依赖的package包/类
/**
 * Return if the specified file is a valid store file or not.
 * @param fileStatus The {@link FileStatus} of the file
 * @return <tt>true</tt> if the file is valid
 */
public static boolean isValid(final FileStatus fileStatus)
    throws IOException {
  final Path p = fileStatus.getPath();

  if (fileStatus.isDir())
    return false;

  // Check for empty hfile. Should never be the case but can happen
  // after data loss in hdfs for whatever reason (upgrade, etc.): HBASE-646
  // NOTE: that the HFileLink is just a name, so it's an empty file.
  if (!HFileLink.isHFileLink(p) && fileStatus.getLen() <= 0) {
    LOG.warn("Skipping " + p + " because it is empty. HBASE-646 DATA LOSS?");
    return false;
  }

  return validateStoreFileName(p.getName());
}
 

import org.apache.hadoop.hbase.io.HFileLink; //导入方法依赖的package包/类
/**
 * Returns the location where the inputPath will be copied.
 *  - hfiles are encoded as hfile links hfile-region-table
 *  - logs are encoded as serverName/logName
 */
private Path getOutputPath(final Path inputPath) throws IOException {
  Path path;
  if (HFileLink.isHFileLink(inputPath) || StoreFile.isReference(inputPath)) {
    String family = inputPath.getParent().getName();
    String table = HFileLink.getReferencedTableName(inputPath.getName());
    String region = HFileLink.getReferencedRegionName(inputPath.getName());
    String hfile = HFileLink.getReferencedHFileName(inputPath.getName());
    path = new Path(table, new Path(region, new Path(family, hfile)));
  } else if (isHLogLinkPath(inputPath)) {
    String logName = inputPath.getName();
    path = new Path(new Path(outputRoot, HConstants.HREGION_OLDLOGDIR_NAME), logName);
  } else {
    path = inputPath;
  }
  return new Path(outputArchive, path);
}
 

import org.apache.hadoop.hbase.io.HFileLink; //导入方法依赖的package包/类
private FileStatus getFileStatus(final FileSystem fs, final Path path) {
  try {
    if (HFileLink.isHFileLink(path) || StoreFile.isReference(path)) {
      HFileLink link = new HFileLink(inputRoot, inputArchive, path);
      return link.getFileStatus(fs);
    } else if (isHLogLinkPath(path)) {
      String serverName = path.getParent().getName();
      String logName = path.getName();
      return new HLogLink(inputRoot, serverName, logName).getFileStatus(fs);
    }
    return fs.getFileStatus(path);
  } catch (IOException e) {
    LOG.warn("Unable to get the status for file=" + path);
    return null;
  }
}
 

import org.apache.hadoop.hbase.io.HFileLink; //导入方法依赖的package包/类
/**
 * Return if the specified file is a valid store file or not.
 * @param fileStatus The {@link FileStatus} of the file
 * @return <tt>true</tt> if the file is valid
 */
public static boolean isValid(final FileStatus fileStatus)
    throws IOException {
  final Path p = fileStatus.getPath();

  if (fileStatus.isDirectory())
    return false;

  // Check for empty hfile. Should never be the case but can happen
  // after data loss in hdfs for whatever reason (upgrade, etc.): HBASE-646
  // NOTE: that the HFileLink is just a name, so it's an empty file.
  if (!HFileLink.isHFileLink(p) && fileStatus.getLen() <= 0) {
    LOG.warn("Skipping " + p + " because it is empty. HBASE-646 DATA LOSS?");
    return false;
  }

  return validateStoreFileName(p.getName());
}
 

import org.apache.hadoop.hbase.io.HFileLink; //导入方法依赖的package包/类
/**
 * Returns the location where the inputPath will be copied.
 *  - hfiles are encoded as hfile links hfile-region-table
 *  - logs are encoded as serverName/logName
 */
private Path getOutputPath(final Path inputPath) throws IOException {
  Path path;
  if (HFileLink.isHFileLink(inputPath) || StoreFileInfo.isReference(inputPath)) {
    String family = inputPath.getParent().getName();
    TableName table =
        HFileLink.getReferencedTableName(inputPath.getName());
    String region = HFileLink.getReferencedRegionName(inputPath.getName());
    String hfile = HFileLink.getReferencedHFileName(inputPath.getName());
    path = new Path(FSUtils.getTableDir(new Path("./"), table),
        new Path(region, new Path(family, hfile)));
  } else if (isHLogLinkPath(inputPath)) {
    String logName = inputPath.getName();
    path = new Path(new Path(outputRoot, HConstants.HREGION_OLDLOGDIR_NAME), logName);
  } else {
    path = inputPath;
  }
  return new Path(outputArchive, path);
}
 

import org.apache.hadoop.hbase.io.HFileLink; //导入方法依赖的package包/类
/**
 * Try to open the "source" file.
 * Throws an IOException if the communication with the inputFs fail or
 * if the file is not found.
 */
private FSDataInputStream openSourceFile(Context context, final Path path) throws IOException {
  try {
    if (HFileLink.isHFileLink(path) || StoreFileInfo.isReference(path)) {
      return new HFileLink(inputRoot, inputArchive, path).open(inputFs);
    } else if (isHLogLinkPath(path)) {
      String serverName = path.getParent().getName();
      String logName = path.getName();
      return new HLogLink(inputRoot, serverName, logName).open(inputFs);
    }
    return inputFs.open(path);
  } catch (IOException e) {
    context.getCounter(Counter.MISSING_FILES).increment(1);
    LOG.error("Unable to open source file=" + path, e);
    throw e;
  }
}
 

import org.apache.hadoop.hbase.io.HFileLink; //导入方法依赖的package包/类
/**
 * Return if the specified file is a valid store file or not.
 * @param fileStatus The {@link FileStatus} of the file
 * @return <tt>true</tt> if the file is valid
 */
public static boolean isValid(final FileStatus fileStatus)
    throws IOException {
  final Path p = fileStatus.getPath();

  if (fileStatus.isDir())
    return false;

  // Check for empty hfile. Should never be the case but can happen
  // after data loss in hdfs for whatever reason (upgrade, etc.): HBASE-646
  // NOTE: that the HFileLink is just a name, so it's an empty file.
  if (!HFileLink.isHFileLink(p) && fileStatus.getLen() <= 0) {
    LOG.warn("Skipping " + p + " beccreateStoreDirause its empty. HBASE-646 DATA LOSS?");
    return false;
  }

  return validateStoreFileName(p.getName());
}
 

import org.apache.hadoop.hbase.io.HFileLink; //导入方法依赖的package包/类
private FileStatus getFileStatus(final FileSystem fs, final Path path) {
  try {
    if (HFileLink.isHFileLink(path) || StoreFileInfo.isReference(path)) {
      HFileLink link = new HFileLink(inputRoot, inputArchive, path);
      return link.getFileStatus(fs);
    } else if (isHLogLinkPath(path)) {
      String serverName = path.getParent().getName();
      String logName = path.getName();
      return new HLogLink(inputRoot, serverName, logName).getFileStatus(fs);
    }
    return fs.getFileStatus(path);
  } catch (IOException e) {
    LOG.warn("Unable to get the status for file=" + path);
    return null;
  }
}
 

import org.apache.hadoop.hbase.io.HFileLink; //导入方法依赖的package包/类
private FSDataInputStream openSourceFile(final Path path) {
  try {
    if (HFileLink.isHFileLink(path) || StoreFile.isReference(path)) {
      return new HFileLink(inputRoot, inputArchive, path).open(inputFs);
    } else if (isHLogLinkPath(path)) {
      String serverName = path.getParent().getName();
      String logName = path.getName();
      return new HLogLink(inputRoot, serverName, logName).open(inputFs);
    }
    return inputFs.open(path);
  } catch (IOException e) {
    LOG.error("Unable to open source file=" + path, e);
    return null;
  }
}
 

import org.apache.hadoop.hbase.io.HFileLink; //导入方法依赖的package包/类
/**
 * Create a Store File Info
 * @param conf the {@link Configuration} to use
 * @param fs The current file system to use.
 * @param fileStatus The {@link FileStatus} of the file
 */
public StoreFileInfo(final Configuration conf, final FileSystem fs, final FileStatus fileStatus)
    throws IOException {
  this.conf = conf;
  this.fileStatus = fileStatus;
  Path p = fileStatus.getPath();
  if (HFileLink.isHFileLink(p)) {
    // HFileLink
    this.reference = null;
    this.link = new HFileLink(conf, p);
    if (LOG.isTraceEnabled()) LOG.trace(p + " is a link");
  } else if (isReference(p)) {
    this.reference = Reference.read(fs, p);
    Path referencePath = getReferredToFile(p);
    if (HFileLink.isHFileLink(referencePath)) {
      // HFileLink Reference
      this.link = new HFileLink(conf, referencePath);
    } else {
      // Reference
      this.link = null;
    }
    if (LOG.isTraceEnabled()) LOG.trace(p + " is a " + reference.getFileRegion() +
      " reference to " + referencePath);
  } else if (isHFile(p)) {
    // HFile
    this.reference = null;
    this.link = null;
  } else {
    throw new IOException("path=" + p + " doesn't look like a valid StoreFile");
  }
}
 

import org.apache.hadoop.hbase.io.HFileLink; //导入方法依赖的package包/类
private void verifyStoreFile(final Path snapshotDir, final HRegionInfo regionInfo,
    final String family, final String fileName) throws IOException {
  Path refPath = null;
  if (StoreFileInfo.isReference(fileName)) {
    // If is a reference file check if the parent file is present in the snapshot
    Path snapshotHFilePath = new Path(new Path(
        new Path(snapshotDir, regionInfo.getEncodedName()), family), fileName);
    refPath = StoreFileInfo.getReferredToFile(snapshotHFilePath);
    if (!fs.exists(refPath)) {
      throw new CorruptedSnapshotException("Missing parent hfile for: " + fileName, snapshot);
    }
  }

  Path linkPath;
  if (refPath != null && HFileLink.isHFileLink(refPath)) {
    linkPath = new Path(family, refPath.getName());
  } else if (HFileLink.isHFileLink(fileName)) {
    linkPath = new Path(family, fileName);
  } else {
    linkPath = new Path(family, HFileLink.createHFileLinkName(tableName,
      regionInfo.getEncodedName(), fileName));
  }

  // check if the linked file exists (in the archive, or in the table dir)
  HFileLink link = new HFileLink(services.getConfiguration(), linkPath);
  if (!link.exists(fs)) {
    throw new CorruptedSnapshotException("Can't find hfile: " + fileName
        + " in the real (" + link.getOriginPath() + ") or archive (" + link.getArchivePath()
        + ") directory for the primary table.", snapshot);
  }
}
 

import org.apache.hadoop.hbase.io.HFileLink; //导入方法依赖的package包/类
private FSDataInputStream openSourceFile(final Path path) {
  try {
    if (HFileLink.isHFileLink(path) || StoreFileInfo.isReference(path)) {
      return new HFileLink(inputRoot, inputArchive, path).open(inputFs);
    } else if (isHLogLinkPath(path)) {
      String serverName = path.getParent().getName();
      String logName = path.getName();
      return new HLogLink(inputRoot, serverName, logName).open(inputFs);
    }
    return inputFs.open(path);
  } catch (IOException e) {
    LOG.error("Unable to open source file=" + path, e);
    return null;
  }
}
 

import org.apache.hadoop.hbase.io.HFileLink; //导入方法依赖的package包/类
/**
 * Prepares the map of files.
 * 
 * @throws IOException
 */
private void init() throws IOException {
  FileStatus[] fileStats = listStatus(familyDir, id.getStartKey());
  if (null == fileStats) {
    return;
  }

  int smallFileCount = 0;
  float invalidFileRatio = conf.getFloat(MobConstants.MOB_COMPACTION_INVALID_FILE_RATIO,
      MobConstants.DEFAULT_MOB_COMPACTION_INVALID_FILE_RATIO);
  long smallFileThreshold = conf.getLong(MobConstants.MOB_COMPACTION_SMALL_FILE_THRESHOLD,
      MobConstants.DEFAULT_MOB_COMPACTION_SMALL_FILE_THRESHOLD);
  // list the files. Just merge the hfiles, don't merge the hfile links.
  // prepare the map of mob files. The key is the file name, the value is the file status.
  // if the mob file is a hfile link, use the name of its referenced file as the key.
  for (FileStatus fileStat : fileStats) {
    MobFileStatus mobFileStatus = null;
    if (HFileLink.isHFileLink(fileStat.getPath())) {
      // Leave the hfile link alone
      HFileLink hfileLink = new HFileLink(conf, fileStat.getPath());
      Path originalPath = hfileLink.getOriginPath();
      mobFileStatus = new MobFileStatus(fileStat, fileStat.getPath().getName());
      // key is file name (not hfile name), value is hfile link status.
      fileStatusMap.put(originalPath.getName(), mobFileStatus);
    } else {
      mobFileStatus = new MobFileStatus(fileStat);
      mobFileStatus.setInvalidFileRatio(invalidFileRatio).setSmallFileThreshold(
          smallFileThreshold);
      if (mobFileStatus.needMerge()) {
        smallFileCount++;
      }
      // key is file name (not hfile name), value is hfile status.
      fileStatusMap.put(fileStat.getPath().getName(), mobFileStatus);
    }
  }
  if (smallFileCount >= 2) {
    // merge the files only when there're more than 1 files in the same partition.
    this.mergeSmall = true;
  }
}
 

import org.apache.hadoop.hbase.io.HFileLink; //导入方法依赖的package包/类
/**
 * This is a helper function to compute HDFS block distribution on demand
 * @param conf configuration
 * @param tableDescriptor TableDescriptor of the table
 * @param regionInfo encoded name of the region
 * @param tablePath the table directory
 * @return The HDFS blocks distribution for the given region.
 * @throws IOException
 */
public static HDFSBlocksDistribution computeHDFSBlocksDistribution(Configuration conf,
    TableDescriptor tableDescriptor, RegionInfo regionInfo, Path tablePath) throws IOException {
  HDFSBlocksDistribution hdfsBlocksDistribution = new HDFSBlocksDistribution();
  FileSystem fs = tablePath.getFileSystem(conf);

  HRegionFileSystem regionFs = new HRegionFileSystem(conf, fs, tablePath, regionInfo);
  for (ColumnFamilyDescriptor family : tableDescriptor.getColumnFamilies()) {
    List<LocatedFileStatus> locatedFileStatusList = HRegionFileSystem
        .getStoreFilesLocatedStatus(regionFs, family.getNameAsString(), true);
    if (locatedFileStatusList == null) {
      continue;
    }

    for (LocatedFileStatus status : locatedFileStatusList) {
      Path p = status.getPath();
      if (StoreFileInfo.isReference(p) || HFileLink.isHFileLink(p)) {
        // Only construct StoreFileInfo object if its not a hfile, save obj
        // creation
        StoreFileInfo storeFileInfo = new StoreFileInfo(conf, fs, status);
        hdfsBlocksDistribution.add(storeFileInfo
            .computeHDFSBlocksDistribution(fs));
      } else if (StoreFileInfo.isHFile(p)) {
        // If its a HFile, then lets just add to the block distribution
        // lets not create more objects here, not even another HDFSBlocksDistribution
        FSUtils.addToHDFSBlocksDistribution(hdfsBlocksDistribution,
            status.getBlockLocations());
      } else {
        throw new IOException("path=" + p
            + " doesn't look like a valid StoreFile");
      }
    }
  }
  return hdfsBlocksDistribution;
}