Java ContentSummary.getLength方法代碼示例

import org.apache.hadoop.fs.ContentSummary; //導入方法依賴的package包/類
/**
 * Estimate the number of reducers needed for this job, based on job input,
 * and configuration parameters.
 *
 * The output of this method should only be used if the output of this
 * MapRedTask is not being used to populate a bucketed table and the user
 * has not specified the number of reducers to use.
 *
 * @return the number of reducers.
 */
public static int estimateNumberOfReducers(HiveConf conf, ContentSummary inputSummary,
                                           MapWork work, boolean finalMapRed) throws IOException {
  long bytesPerReducer = conf.getLongVar(HiveConf.ConfVars.BYTESPERREDUCER);
  int maxReducers = conf.getIntVar(HiveConf.ConfVars.MAXREDUCERS);

  double samplePercentage = getHighestSamplePercentage(work);
  long totalInputFileSize = getTotalInputFileSize(inputSummary, work, samplePercentage);

  // if all inputs are sampled, we should shrink the size of reducers accordingly.
  if (totalInputFileSize != inputSummary.getLength()) {
    LOG.info("BytesPerReducer=" + bytesPerReducer + " maxReducers="
        + maxReducers + " estimated totalInputFileSize=" + totalInputFileSize);
  } else {
    LOG.info("BytesPerReducer=" + bytesPerReducer + " maxReducers="
      + maxReducers + " totalInputFileSize=" + totalInputFileSize);
  }

  // If this map reduce job writes final data to a table and bucketing is being inferred,
  // and the user has configured Hive to do this, make sure the number of reducers is a
  // power of two
  boolean powersOfTwo = conf.getBoolVar(HiveConf.ConfVars.HIVE_INFER_BUCKET_SORT_NUM_BUCKETS_POWER_TWO) &&
      finalMapRed && !work.getBucketedColsByDirectory().isEmpty();

  return estimateReducers(totalInputFileSize, bytesPerReducer, maxReducers, powersOfTwo);
}
 

import org.apache.hadoop.fs.ContentSummary; //導入方法依賴的package包/類
@Override
protected void processPath(PathData item) throws IOException {
  ContentSummary contentSummary = item.fs.getContentSummary(item.path);
  long length = contentSummary.getLength();
  long spaceConsumed = contentSummary.getSpaceConsumed();
  usagesTable.addRow(formatSize(length), formatSize(spaceConsumed), item);
}
 

import org.apache.hadoop.fs.ContentSummary; //導入方法依賴的package包/類
public static boolean isEmptyPath(JobConf job, Path dirPath, Context ctx)
    throws Exception {
  if (ctx != null) {
    ContentSummary cs = ctx.getCS(dirPath);
    if (cs != null) {
      LOG.info("Content Summary " + dirPath + "length: " + cs.getLength() + " num files: "
          + cs.getFileCount() + " num directories: " + cs.getDirectoryCount());
      return (cs.getLength() == 0 && cs.getFileCount() == 0 && cs.getDirectoryCount() <= 1);
    } else {
      LOG.info("Content Summary not cached for " + dirPath);
    }
  }
  return isEmptyPath(job, dirPath);
}
 

import org.apache.hadoop.fs.ContentSummary; //導入方法依賴的package包/類
/**
 * Computes the total input file size. If block sampling was used it will scale this
 * value by the highest sample percentage (as an estimate for input).
 *
 * @param inputSummary
 * @param work
 * @param highestSamplePercentage
 * @return estimated total input size for job
 */
public static long getTotalInputFileSize (ContentSummary inputSummary, MapWork work,
    double highestSamplePercentage) {
  long totalInputFileSize = inputSummary.getLength();
  if (work.getNameToSplitSample() == null || work.getNameToSplitSample().isEmpty()) {
    // If percentage block sampling wasn't used, we don't need to do any estimation
    return totalInputFileSize;
  }

  if (highestSamplePercentage >= 0) {
    totalInputFileSize = Math.min((long) (totalInputFileSize * (highestSamplePercentage / 100D))
        , totalInputFileSize);
  }
  return totalInputFileSize;
}