Java ContextUtil.getConfiguration方法代码示例

import parquet.hadoop.util.ContextUtil; //导入方法依赖的package包/类
/**
 * {@inheritDoc}
 */
@Override
public List<InputSplit> getSplits(JobContext jobContext) throws IOException {
    Configuration configuration = ContextUtil.getConfiguration(jobContext);
    List<InputSplit> splits = new ArrayList<InputSplit>();

    if (isTaskSideMetaData(configuration)) {
        // Although not required by the API, some clients may depend on always
        // receiving ParquetInputSplit. Translation is required at some point.
        for (InputSplit split : super.getSplits(jobContext)) {
            Preconditions.checkArgument(split instanceof FileSplit,
                    "Cannot wrap non-FileSplit: " + split);
            splits.add(ParquetInputSplit.from((FileSplit) split));
        }
        return splits;

    } else {
        splits.addAll(getSplits(configuration, getFooters(jobContext)));
    }

    return splits;
}
 

import parquet.hadoop.util.ContextUtil; //导入方法依赖的package包/类
@Override
public void commitJob(JobContext jobContext) throws IOException {
    super.commitJob(jobContext);
    Configuration conf = ContextUtil.getConfiguration(jobContext);
    Path outputPath = FileOutputFormat.getOutputPath(new JobConf(conf));
    ParquetOutputCommitter.writeMetaDataFile(conf, outputPath);
}
 

import parquet.hadoop.util.ContextUtil; //导入方法依赖的package包/类
public static void setUnboundRecordFilter(Job job, Class<? extends UnboundRecordFilter> filterClass) {
    Configuration conf = ContextUtil.getConfiguration(job);
    checkArgument(getFilterPredicate(conf) == null,
            "You cannot provide an UnboundRecordFilter after providing a FilterPredicate");

    conf.set(UNBOUND_RECORD_FILTER, filterClass.getName());
}
 

import parquet.hadoop.util.ContextUtil; //导入方法依赖的package包/类
/**
 * {@inheritDoc}
 */
@Override
public RecordReader<Void, T> createRecordReader(
        InputSplit inputSplit,
        TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException {
    Configuration conf = ContextUtil.getConfiguration(taskAttemptContext);
    ReadSupport<T> readSupport = getReadSupport(conf);
    return new ParquetRecordReader<T>(readSupport, getFilter(conf));
}
 

import parquet.hadoop.util.ContextUtil; //导入方法依赖的package包/类
public void commitJob(JobContext jobContext) throws IOException {
    super.commitJob(jobContext);
    Configuration configuration = ContextUtil.getConfiguration(jobContext);
    writeMetaDataFile(configuration, outputPath);
}
 

import parquet.hadoop.util.ContextUtil; //导入方法依赖的package包/类
/**
 * @param jobContext the current job context
 * @return the footers for the files
 * @throws IOException
 */
public List<Footer> getFooters(JobContext jobContext) throws IOException {
    List<FileStatus> statuses = listStatus(jobContext);
    if (statuses.isEmpty()) {
        return Collections.emptyList();
    }
    Configuration config = ContextUtil.getConfiguration(jobContext);
    List<Footer> footers = new ArrayList<Footer>(statuses.size());
    Set<FileStatus> missingStatuses = new HashSet<FileStatus>();
    Map<Path, FileStatusWrapper> missingStatusesMap =
            new HashMap<Path, FileStatusWrapper>(missingStatuses.size());

    if (footersCache == null) {
        footersCache =
                new LruCache<FileStatusWrapper, FootersCacheValue>(Math.max(statuses.size(), MIN_FOOTER_CACHE_SIZE));
    }
    for (FileStatus status : statuses) {
        FileStatusWrapper statusWrapper = new FileStatusWrapper(status);
        FootersCacheValue cacheEntry =
                footersCache.getCurrentValue(statusWrapper);
        if (Log.DEBUG) {
            LOG.debug("Cache entry " + (cacheEntry == null ? "not " : "")
                    + " found for '" + status.getPath() + "'");
        }
        if (cacheEntry != null) {
            footers.add(cacheEntry.getFooter());
        } else {
            missingStatuses.add(status);
            missingStatusesMap.put(status.getPath(), statusWrapper);
        }
    }
    if (Log.DEBUG) {
        LOG.debug("found " + footers.size() + " footers in cache and adding up "
                + "to " + missingStatuses.size() + " missing footers to the cache");
    }


    if (missingStatuses.isEmpty()) {
        return footers;
    }

    List<Footer> newFooters = getFooters(config, missingStatuses);
    for (Footer newFooter : newFooters) {
        // Use the original file status objects to make sure we store a
        // conservative (older) modification time (i.e. in case the files and
        // footers were modified and it's not clear which version of the footers
        // we have)
        FileStatusWrapper fileStatus = missingStatusesMap.get(newFooter.getFile());
        footersCache.put(fileStatus, new FootersCacheValue(fileStatus, newFooter));
    }

    footers.addAll(newFooters);
    return footers;
}
 

import parquet.hadoop.util.ContextUtil; //导入方法依赖的package包/类
@Override
public Configuration getConfiguration() {
    return ContextUtil.getConfiguration(context);
}