Java StoreUtils类代码示例

import org.apache.hadoop.hbase.regionserver.StoreUtils; //导入依赖的package包/类
@Override
public CompactionRequest selectCompaction(Collection<StoreFile> candidateFiles,
    List<StoreFile> filesCompacting, boolean isUserCompaction, boolean mayUseOffPeak,
    boolean forceMajor) throws IOException {
  
  if(forceMajor){
    LOG.warn("Major compaction is not supported for FIFO compaction policy. Ignore the flag.");
  }
  boolean isAfterSplit = StoreUtils.hasReferences(candidateFiles);
  if(isAfterSplit){
    LOG.info("Split detected, delegate selection to the parent policy.");
    return super.selectCompaction(candidateFiles, filesCompacting, isUserCompaction, 
      mayUseOffPeak, forceMajor);
  }
  
  // Nothing to compact
  Collection<StoreFile> toCompact = getExpiredStores(candidateFiles, filesCompacting);
  CompactionRequest result = new CompactionRequest(toCompact);
  return result;
}
 

import org.apache.hadoop.hbase.regionserver.StoreUtils; //导入依赖的package包/类
/**
 * @param filesToCompact
 * @return When to run next major compaction
 */
public long getNextMajorCompactTime(final Collection<StoreFile> filesToCompact) {
  // default = 24hrs
  long ret = comConf.getMajorCompactionPeriod();
  if (ret > 0) {
    // default = 20% = +/- 4.8 hrs
    double jitterPct = comConf.getMajorCompactionJitter();
    if (jitterPct > 0) {
      long jitter = Math.round(ret * jitterPct);
      // deterministic jitter avoids a major compaction storm on restart
      Integer seed = StoreUtils.getDeterministicRandomSeed(filesToCompact);
      if (seed != null) {
        // Synchronized to ensure one user of random instance at a time.
        double rnd = -1;
        synchronized (this) {
          this.random.setSeed(seed);
          rnd = this.random.nextDouble();
        }
        ret += jitter - Math.round(2L * jitter * rnd);
      } else {
        ret = 0; // If seed is null, then no storefiles == no major compaction
      }
    }
  }
  return ret;
}
 

import org.apache.hadoop.hbase.regionserver.StoreUtils; //导入依赖的package包/类
public long getNextMajorCompactTime(final Collection<StoreFile> filesToCompact) {
  // default = 24hrs
  long ret = comConf.getMajorCompactionPeriod();
  if (ret > 0) {
    // default = 20% = +/- 4.8 hrs
    double jitterPct = comConf.getMajorCompactionJitter();
    if (jitterPct > 0) {
      long jitter = Math.round(ret * jitterPct);
      // deterministic jitter avoids a major compaction storm on restart
      Integer seed = StoreUtils.getDeterministicRandomSeed(filesToCompact);
      if (seed != null) {
        double rnd = (new Random(seed)).nextDouble();
        ret += jitter - Math.round(2L * jitter * rnd);
      } else {
        ret = 0; // no storefiles == no major compaction
      }
    }
  }
  return ret;
}
 

import org.apache.hadoop.hbase.regionserver.StoreUtils; //导入依赖的package包/类
@Override
public CompactionRequestImpl selectCompaction(Collection<HStoreFile> candidateFiles,
    List<HStoreFile> filesCompacting, boolean isUserCompaction, boolean mayUseOffPeak,
    boolean forceMajor) throws IOException {
  if(forceMajor){
    LOG.warn("Major compaction is not supported for FIFO compaction policy. Ignore the flag.");
  }
  boolean isAfterSplit = StoreUtils.hasReferences(candidateFiles);
  if(isAfterSplit){
    LOG.info("Split detected, delegate selection to the parent policy.");
    return super.selectCompaction(candidateFiles, filesCompacting, isUserCompaction, 
      mayUseOffPeak, forceMajor);
  }

  // Nothing to compact
  Collection<HStoreFile> toCompact = getExpiredStores(candidateFiles, filesCompacting);
  CompactionRequestImpl result = new CompactionRequestImpl(toCompact);
  return result;
}
 

import org.apache.hadoop.hbase.regionserver.StoreUtils; //导入依赖的package包/类
public long getNextMajorCompactTime(final List<StoreFile> filesToCompact) {
  // default = 24hrs
  long ret = comConf.getMajorCompactionPeriod();
  if (ret > 0) {
    // default = 20% = +/- 4.8 hrs
    double jitterPct = comConf.getMajorCompactionJitter();
    if (jitterPct > 0) {
      long jitter = Math.round(ret * jitterPct);
      // deterministic jitter avoids a major compaction storm on restart
      Integer seed = StoreUtils.getDeterministicRandomSeed(filesToCompact);
      if (seed != null) {
        double rnd = (new Random(seed)).nextDouble();
        ret += jitter - Math.round(2L * jitter * rnd);
      } else {
        ret = 0; // no storefiles == no major compaction
      }
    }
  }
  return ret;
}
 

import org.apache.hadoop.hbase.regionserver.StoreUtils; //导入依赖的package包/类
@Override
public boolean isMajorCompaction(Collection<StoreFile> filesToCompact) throws IOException {
  boolean isAfterSplit = StoreUtils.hasReferences(filesToCompact);
  if(isAfterSplit){
    LOG.info("Split detected, delegate to the parent policy.");
    return super.isMajorCompaction(filesToCompact);
  }
  return false;
}
 

import org.apache.hadoop.hbase.regionserver.StoreUtils; //导入依赖的package包/类
@Override
public boolean needsCompaction(Collection<StoreFile> storeFiles, 
    List<StoreFile> filesCompacting) {  
  boolean isAfterSplit = StoreUtils.hasReferences(storeFiles);
  if(isAfterSplit){
    LOG.info("Split detected, delegate to the parent policy.");
    return super.needsCompaction(storeFiles, filesCompacting);
  }
  return hasExpiredStores(storeFiles);
}
 

import org.apache.hadoop.hbase.regionserver.StoreUtils; //导入依赖的package包/类
public boolean needsCompactions(StripeInformationProvider si, List<StoreFile> filesCompacting) {
  // Approximation on whether we need compaction.
  return filesCompacting.isEmpty()
      && (StoreUtils.hasReferences(si.getStorefiles())
        || (si.getLevel0Files().size() >= this.config.getLevel0MinFiles())
        || needsSingleStripeCompaction(si));
}
 

import org.apache.hadoop.hbase.regionserver.StoreUtils; //导入依赖的package包/类
private boolean needEmptyFile(CompactionRequestImpl request) {
  // if we are going to compact the last N files, then we need to emit an empty file to retain the
  // maxSeqId if we haven't written out anything.
  OptionalLong maxSeqId = StoreUtils.getMaxSequenceIdInList(request.getFiles());
  OptionalLong storeMaxSeqId = store.getMaxSequenceId();
  return maxSeqId.isPresent() && storeMaxSeqId.isPresent() &&
      maxSeqId.getAsLong() == storeMaxSeqId.getAsLong();
}
 

import org.apache.hadoop.hbase.regionserver.StoreUtils; //导入依赖的package包/类
/**
 * @param filesToCompact
 * @return When to run next major compaction
 */
public long getNextMajorCompactTime(Collection<HStoreFile> filesToCompact) {
  /** Default to {@link org.apache.hadoop.hbase.HConstants#DEFAULT_MAJOR_COMPACTION_PERIOD}. */
  long period = comConf.getMajorCompactionPeriod();
  if (period <= 0) {
    return period;
  }

  /**
   * Default to {@link org.apache.hadoop.hbase.HConstants#DEFAULT_MAJOR_COMPACTION_JITTER},
   * that is, +/- 3.5 days (7 days * 0.5).
   */
  double jitterPct = comConf.getMajorCompactionJitter();
  if (jitterPct <= 0) {
    return period;
  }

  // deterministic jitter avoids a major compaction storm on restart
  OptionalInt seed = StoreUtils.getDeterministicRandomSeed(filesToCompact);
  if (seed.isPresent()) {
    // Synchronized to ensure one user of random instance at a time.
    double rnd;
    synchronized (this) {
      this.random.setSeed(seed.getAsInt());
      rnd = this.random.nextDouble();
    }
    long jitter = Math.round(period * jitterPct);
    return period + jitter - Math.round(2L * jitter * rnd);
  } else {
    return 0L;
  }
}
 

import org.apache.hadoop.hbase.regionserver.StoreUtils; //导入依赖的package包/类
@Override
public boolean shouldPerformMajorCompaction(Collection<HStoreFile> filesToCompact)
  throws IOException {
  boolean isAfterSplit = StoreUtils.hasReferences(filesToCompact);
  if(isAfterSplit){
    LOG.info("Split detected, delegate to the parent policy.");
    return super.shouldPerformMajorCompaction(filesToCompact);
  }
  return false;
}
 

import org.apache.hadoop.hbase.regionserver.StoreUtils; //导入依赖的package包/类
@Override
public boolean needsCompaction(Collection<HStoreFile> storeFiles,
    List<HStoreFile> filesCompacting) {
  boolean isAfterSplit = StoreUtils.hasReferences(storeFiles);
  if(isAfterSplit){
    LOG.info("Split detected, delegate to the parent policy.");
    return super.needsCompaction(storeFiles, filesCompacting);
  }
  return hasExpiredStores(storeFiles);
}
 

import org.apache.hadoop.hbase.regionserver.StoreUtils; //导入依赖的package包/类
public boolean needsCompactions(StripeInformationProvider si, List<HStoreFile> filesCompacting) {
  // Approximation on whether we need compaction.
  return filesCompacting.isEmpty()
      && (StoreUtils.hasReferences(si.getStorefiles())
        || (si.getLevel0Files().size() >= this.config.getLevel0MinFiles())
        || needsSingleStripeCompaction(si));
}
 

import org.apache.hadoop.hbase.regionserver.StoreUtils; //导入依赖的package包/类
/**
 * @param candidateFiles candidate files, ordered from oldest to newest
 * @return subset copy of candidate list that meets compaction criteria
 * @throws java.io.IOException
 */
public CompactSelection selectCompaction(List<StoreFile> candidateFiles,
    boolean isUserCompaction, boolean forceMajor)
  throws IOException {
  // Prelimanry compaction subject to filters
  CompactSelection candidateSelection = new CompactSelection(candidateFiles);
  long cfTtl = this.storeConfig.getStoreFileTtl();
  if (!forceMajor) {
    // If there are expired files, only select them so that compaction deletes them
    if (comConf.shouldDeleteExpired() && (cfTtl != Long.MAX_VALUE)) {
      CompactSelection expiredSelection = selectExpiredStoreFiles(
        candidateSelection, EnvironmentEdgeManager.currentTimeMillis() - cfTtl);
      if (expiredSelection != null) {
        return expiredSelection;
      }
    }
    candidateSelection = skipLargeFiles(candidateSelection);
  }

  // Force a major compaction if this is a user-requested major compaction,
  // or if we do not have too many files to compact and this was requested
  // as a major compaction.
  // Or, if there are any references among the candidates.
  boolean majorCompaction = (
    (forceMajor && isUserCompaction)
    || ((forceMajor || isMajorCompaction(candidateSelection.getFilesToCompact()))
        && (candidateSelection.getFilesToCompact().size() < comConf.getMaxFilesToCompact()))
    || StoreUtils.hasReferences(candidateSelection.getFilesToCompact())
    );

  if (!majorCompaction) {
    // we're doing a minor compaction, let's see what files are applicable
    candidateSelection = filterBulk(candidateSelection);
    candidateSelection = applyCompactionPolicy(candidateSelection);
    candidateSelection = checkMinFilesCriteria(candidateSelection);
  }
  candidateSelection =
      removeExcessFiles(candidateSelection, isUserCompaction, majorCompaction);
  return candidateSelection;
}
 

import org.apache.hadoop.hbase.regionserver.StoreUtils; //导入依赖的package包/类
/**
 * @param candidateFiles candidate files, ordered from oldest to newest. All files in store.
 * @return subset copy of candidate list that meets compaction criteria
 * @throws java.io.IOException
 */
public CompactionRequest selectCompaction(Collection<StoreFile> candidateFiles,
    final List<StoreFile> filesCompacting, final boolean isUserCompaction,
    final boolean mayUseOffPeak, final boolean forceMajor) throws IOException {
  // Preliminary compaction subject to filters
  ArrayList<StoreFile> candidateSelection = new ArrayList<StoreFile>(candidateFiles);
  // Stuck and not compacting enough (estimate). It is not guaranteed that we will be
  // able to compact more if stuck and compacting, because ratio policy excludes some
  // non-compacting files from consideration during compaction (see getCurrentEligibleFiles).
  int futureFiles = filesCompacting.isEmpty() ? 0 : 1;
  boolean mayBeStuck = (candidateFiles.size() - filesCompacting.size() + futureFiles)
      >= storeConfigInfo.getBlockingFileCount();
  candidateSelection = getCurrentEligibleFiles(candidateSelection, filesCompacting);
  LOG.debug("Selecting compaction from " + candidateFiles.size() + " store files, " +
      filesCompacting.size() + " compacting, " + candidateSelection.size() +
      " eligible, " + storeConfigInfo.getBlockingFileCount() + " blocking");

  // If we can't have all files, we cannot do major anyway
  boolean isAllFiles = candidateFiles.size() == candidateSelection.size();
  if (!(forceMajor && isAllFiles)) {
    candidateSelection = skipLargeFiles(candidateSelection, mayUseOffPeak);
    isAllFiles = candidateFiles.size() == candidateSelection.size();
  }

  // Try a major compaction if this is a user-requested major compaction,
  // or if we do not have too many files to compact and this was requested as a major compaction
  boolean isTryingMajor = (forceMajor && isAllFiles && isUserCompaction)
      || (((forceMajor && isAllFiles) || isMajorCompaction(candidateSelection))
        && (candidateSelection.size() < comConf.getMaxFilesToCompact()));
  // Or, if there are any references among the candidates.
  boolean isAfterSplit = StoreUtils.hasReferences(candidateSelection);
  if (!isTryingMajor && !isAfterSplit) {
    // We're are not compacting all files, let's see what files are applicable
    candidateSelection = filterBulk(candidateSelection);
    candidateSelection = applyCompactionPolicy(candidateSelection, mayUseOffPeak, mayBeStuck);
    candidateSelection = checkMinFilesCriteria(candidateSelection);
  }
  candidateSelection = removeExcessFiles(candidateSelection, isUserCompaction, isTryingMajor);
  // Now we have the final file list, so we can determine if we can do major/all files.
  isAllFiles = (candidateFiles.size() == candidateSelection.size());
  CompactionRequest result = new CompactionRequest(candidateSelection);
  result.setOffPeak(!candidateSelection.isEmpty() && !isAllFiles && mayUseOffPeak);
  result.setIsMajor(isTryingMajor && isAllFiles, isAllFiles);
  return result;
}