當前位置: 首頁>>代碼示例>>Java>>正文

Java MultipleOutputs類代碼示例

本文整理匯總了Java中org.apache.hadoop.mapreduce.lib.output.MultipleOutputs類的典型用法代碼示例。如果您正苦於以下問題：Java MultipleOutputs類的具體用法？Java MultipleOutputs怎麽用？Java MultipleOutputs使用的例子？那麽, 這裏精選的類代碼示例或許可以為您提供幫助。


MultipleOutputs類屬於org.apache.hadoop.mapreduce.lib.output包，在下文中一共展示了MultipleOutputs類的15個代碼示例，這些例子默認根據受歡迎程度排序。您可以為喜歡或者感覺有用的代碼點讚，您的評價將有助於係統推薦出更棒的Java代碼示例。

示例1: reduce

import org.apache.hadoop.mapreduce.lib.output.MultipleOutputs; //導入依賴的package包/類
/**
 * {@inheritDoc}
 */
protected void reduce(final Text key, final Iterable<Text> values, final Context context) throws IOException, InterruptedException {
    final Configuration configuration = context.getConfiguration();
    final String sourcePath = configuration.get("compactionSourcePath");
    final String targetPath = configuration.get("compactionTargetPath");

    // Reducer stores data at the target directory retaining the directory structure of files
    String filePath = key.toString().replace(sourcePath, targetPath);
    if (key.toString().endsWith("/")) {
        filePath = filePath.concat("file");
    }

    log.info("Compaction output path {}", filePath);
    final URI uri = URI.create(filePath);
    final MultipleOutputs multipleOutputs = new MultipleOutputs<NullWritable, Text>(context);
    try {
        for (final Text text : values) {
            multipleOutputs.write(NullWritable.get(), text, uri.toString());
        }
    } finally {
        multipleOutputs.close();
    }
}
開發者ID:ExpediaInceCommercePlatform，項目名稱:dataSqueeze，代碼行數:26，代碼來源:TextCompactionReducer.java

示例2: reduce

import org.apache.hadoop.mapreduce.lib.output.MultipleOutputs; //導入依賴的package包/類
/**
 * {@inheritDoc}
 */
protected void reduce(final Text key, final Iterable<OrcValue> values, final Context context) throws IOException, InterruptedException {
    final Configuration configuration = context.getConfiguration();
    final String sourcePath = configuration.get("compactionSourcePath");
    final String targetPath = configuration.get("compactionTargetPath");

    // Reducer stores data at the target directory retaining the directory structure of files
    String filePath = key.toString().replace(sourcePath, targetPath);
    if (key.toString().endsWith("/")) {
        filePath = filePath.concat("file");
    }

    log.info("Compaction output path {}", filePath);
    final URI uri = URI.create(filePath);
    final MultipleOutputs multipleOutputs = new MultipleOutputs<NullWritable, OrcValue>(context);
    try {
        for (final OrcValue text : values) {
            multipleOutputs.write(NullWritable.get(), text, uri.toString());
        }
    } finally {
        multipleOutputs.close();
    }
}
開發者ID:ExpediaInceCommercePlatform，項目名稱:dataSqueeze，代碼行數:26，代碼來源:OrcCompactionReducer.java

示例3: reduce

import org.apache.hadoop.mapreduce.lib.output.MultipleOutputs; //導入依賴的package包/類
/**
 * {@inheritDoc}
 */
protected void reduce(final Text key, final Iterable<BytesWritable> values, final Context context) throws IOException, InterruptedException {
    final Configuration configuration = context.getConfiguration();
    final String sourcePath = configuration.get("compactionSourcePath");
    final String targetPath = configuration.get("compactionTargetPath");

    // Reducer stores data at the target directory retaining the directory structure of files
    String filePath = key.toString().replace(sourcePath, targetPath);
    if (key.toString().endsWith("/")) {
        filePath = filePath.concat("file");
    }

    log.info("Compaction output path {}", filePath);
    final URI uri = URI.create(filePath);
    final MultipleOutputs multipleOutputs = new MultipleOutputs<NullWritable, BytesWritable>(context);
    try {
        for (final BytesWritable text : values) {
            multipleOutputs.write(NullWritable.get(), text, uri.toString());
        }
    } finally {
        multipleOutputs.close();
    }
}
開發者ID:ExpediaInceCommercePlatform，項目名稱:dataSqueeze，代碼行數:26，代碼來源:BytesWritableCompactionReducer.java

示例4: setup

import org.apache.hadoop.mapreduce.lib.output.MultipleOutputs; //導入依賴的package包/類
public void setup(Context context) throws IOException, InterruptedException {
	Configuration conf = context.getConfiguration();

	 multipleOutputs = new MultipleOutputs(context);
	lowerBoundary = conf.get("LOWER_DATE");
	upperBoundary = conf.get("HIGHER_DATE");

}
開發者ID:gatripat，項目名稱:InsAdjustment，代碼行數:9，代碼來源:CSVparserMapper.java

示例5: writeSingleWARCWritableToOutput

import org.apache.hadoop.mapreduce.lib.output.MultipleOutputs; //導入依賴的package包/類
/**
 * Writes single WARCWritable to the output with specific output file prefix
 *
 * @param warcWritable    warc record
 * @param multipleOutputs output
 * @throws IOException          exception
 * @throws InterruptedException exception
 */
// TODO move somewhere else?
public static void writeSingleWARCWritableToOutput(WARCWritable warcWritable,
        MultipleOutputs<NullWritable, WARCWritable> multipleOutputs)
        throws IOException, InterruptedException
{
    WARCRecord.Header header = warcWritable.getRecord().getHeader();
    String license = header.getField(WARCRecord.WARCRecordFieldConstants.LICENSE);
    String language = header.getField(WARCRecord.WARCRecordFieldConstants.LANGUAGE);
    String noBoilerplate = header
            .getField(WARCRecord.WARCRecordFieldConstants.NO_BOILERPLATE);
    String minimalHtml = header.getField(WARCRecord.WARCRecordFieldConstants.MINIMAL_HTML);

    // set the file name prefix
    String fileName = createOutputFilePrefix(license, language, noBoilerplate, minimalHtml);

    // bottleneck of single reducer for all "Lic_none_Lang_en" pages (majority of Web)
    //        if ("en".equals(language) && LicenseDetector.NO_LICENCE.equals(license)) {
    //            long simHash = Long
    //                    .valueOf(header.getField(WARCRecord.WARCRecordFieldConstants.SIMHASH));
    //            int binNumber = getBinNumberFromSimHash(simHash);
    //            fileName = createOutputFilePrefix(license, language, noBoilerplate);
    //        }

    multipleOutputs.write(NullWritable.get(), warcWritable, fileName);
}
開發者ID:dkpro，項目名稱:dkpro-c4corpus，代碼行數:34，代碼來源:WARCWriterReducerClass.java

示例6: setup

import org.apache.hadoop.mapreduce.lib.output.MultipleOutputs; //導入依賴的package包/類
@Override
public void setup(Context ctx) throws IOException, InterruptedException
{
  super.setup(ctx);

  mos = new MultipleOutputs<>(ctx);

  FileSystem fs = FileSystem.newInstance(ctx.getConfiguration());
  storage = new HadoopFileSystemStore(fs);
  String queryDir = ctx.getConfiguration().get("pirMR.queryInputDir");
  Query query = storage.recall(queryDir, Query.class);
  QueryInfo queryInfo = query.getQueryInfo();

  outputFile = ctx.getConfiguration().get("pirMR.outputFile");

  response = new Response(queryInfo);
}
開發者ID:apache，項目名稱:incubator-pirk，代碼行數:18，代碼來源:FinalResponseReducer.java

示例7: writeTopKBounds

import org.apache.hadoop.mapreduce.lib.output.MultipleOutputs; //導入依賴的package包/類
public void writeTopKBounds(MultipleOutputs<?, ?> sideOutputs, String outputName, String path, int minsup) throws IOException, InterruptedException {
	final IntWritable itemW  = new IntWritable();
	final IntWritable boundW = new IntWritable();
	
	TIntObjectIterator<PatternWithFreq[]> it = this.topK.iterator();
	
	while (it.hasNext()) {
		it.advance();
		if (it.value()[this.k - 1] != null) {
			final int supportCount = it.value()[this.k - 1].getSupportCount();
			
			if (supportCount > minsup) {
				itemW.set(it.key());
				boundW.set(supportCount);
				sideOutputs.write(outputName, itemW, boundW, path);
			}
		}
	}
}
開發者ID:slide-lig，項目名稱:TopPI，代碼行數:20，代碼來源:PerItemTopKHadoopCollector.java

示例8: setup

import org.apache.hadoop.mapreduce.lib.output.MultipleOutputs; //導入依賴的package包/類
@Override
protected void setup(Context context) throws IOException, InterruptedException {
	Configuration conf = context.getConfiguration();
	this.reverseRebasing = DistCache.readReverseRebasing(DistCache.getCachedFiles(context), conf);

	if (conf.get(MinerWrapper.KEY_BOUNDS_PATH) != null) {
		this.sideOutputs = new MultipleOutputs<IntWritable, SupportAndTransactionWritable>(context);
	}
	this.manyItems = conf.getBoolean(TopPIoverHadoop.KEY_MANY_ITEMS_MODE, false);
	if (manyItems) {
		this.marker = TopPIoverHadoop.FILTERED_DIRNAME;
	} else {
		this.marker = conf.get(TopPIoverHadoop.KEY_INPUT);
		String[] sp = marker.split("/");
		if (sp.length > 2) {
			this.marker = sp[sp.length - 1];
		}
	}
	this.minsup = conf.getInt(TopPIoverHadoop.KEY_MINSUP, 10);
	this.nbGroups = conf.getInt(TopPIoverHadoop.KEY_NBGROUPS, 1);
	this.maxItemId = conf.getInt(TopPIoverHadoop.KEY_REBASING_MAX_ID, 1);
	this.k = conf.getInt(TopPIoverHadoop.KEY_K, 1);
}
開發者ID:slide-lig，項目名稱:TopPI，代碼行數:24，代碼來源:MiningReducer.java

示例9: configureTextOutput

import org.apache.hadoop.mapreduce.lib.output.MultipleOutputs; //導入依賴的package包/類
/**
 * Set up a MapReduce job to output human-readable text.
 */
protected void configureTextOutput(String destination) {
    Path outPath;
    outPath = MRReasoningUtils.getOutputPath(job.getConfiguration(), destination);
    TextOutputFormat.setOutputPath(job, outPath);
    LazyOutputFormat.setOutputFormatClass(job, TextOutputFormat.class);
    MultipleOutputs.addNamedOutput(job, MRReasoningUtils.INTERMEDIATE_OUT,
        TextOutputFormat.class, NullWritable.class, Text.class);
    MultipleOutputs.addNamedOutput(job, MRReasoningUtils.TERMINAL_OUT,
        TextOutputFormat.class, NullWritable.class, Text.class);
    MultipleOutputs.addNamedOutput(job, MRReasoningUtils.SCHEMA_OUT,
        TextOutputFormat.class, NullWritable.class, Text.class);
    MultipleOutputs.addNamedOutput(job, MRReasoningUtils.INCONSISTENT_OUT,
        TextOutputFormat.class, NullWritable.class, Text.class);
    MultipleOutputs.addNamedOutput(job, MRReasoningUtils.DEBUG_OUT,
        TextOutputFormat.class, Text.class, Text.class);
    MultipleOutputs.setCountersEnabled(job, true);
}
開發者ID:apache，項目名稱:incubator-rya，代碼行數:21，代碼來源:AbstractReasoningTool.java

示例10: verifyNamedOutput

import org.apache.hadoop.mapreduce.lib.output.MultipleOutputs; //導入依賴的package包/類
protected void verifyNamedOutput(MultipleOutputs multiOut, String name, Object key, Object value, String path, VerificationMode mode) {
	ArgumentCaptor keyArg = ArgumentCaptor.forClass(key.getClass());
	ArgumentCaptor valueArg = ArgumentCaptor.forClass(value.getClass());
	try {
		if (name == null) {
			verify(multiOut, mode).write(keyArg.capture(), valueArg.capture(), path);
		}
		else {
			if (path == null) {
				verify(multiOut, mode).write(eq(name), keyArg.capture(), valueArg.capture());
				assertEquals(key, keyArg.getValue());
				assertEquals(value, valueArg.getValue());
			}
			else {
				verify(multiOut, mode).write(name, keyArg.capture(), valueArg.capture(), path);
			}
		}
	} catch (IOException | InterruptedException e) {
		fail(e.getMessage());
	}
}
開發者ID:conversant，項目名稱:mara，代碼行數:22，代碼來源:BaseMRUnitTest.java

示例11: getGenericTypeParams

import org.apache.hadoop.mapreduce.lib.output.MultipleOutputs; //導入依賴的package包/類
/**
 * If this is a multiple output we're annotating, see if there are type parameters to
 * use for the key/value classes.
 * 
 * @param target	object to reflect for type params
 * @return			the key/value type parameters
 */
protected Pair<Type, Type> getGenericTypeParams(Object target) {
	Pair<Type, Type> kvTypePair = null;
	if (target instanceof Field) {
		Field field = (Field)target;
		if (field.getType() == MultipleOutputs.class) {
			Type genericType = field.getGenericType();
			if (genericType instanceof ParameterizedType) {
				Type[] keyValueTypes = ((ParameterizedType)genericType).getActualTypeArguments();
				if (keyValueTypes != null && keyValueTypes.length == 2) {
					kvTypePair = new ImmutablePair<>(keyValueTypes[0], keyValueTypes[1]);
				}
			}
		}
	}
	return kvTypePair;
}
開發者ID:conversant，項目名稱:mara，代碼行數:24，代碼來源:NamedOutputAnnotationHandler.java

示例12: setup

import org.apache.hadoop.mapreduce.lib.output.MultipleOutputs; //導入依賴的package包/類
@Override
public void setup(Context context) throws IOException, InterruptedException {
  LOGGER.info("RollupPhaseOneJob.RollupPhaseOneMapper.setup()");
  mos = new MultipleOutputs<BytesWritable, BytesWritable>(context);
  Configuration configuration = context.getConfiguration();
  FileSystem fileSystem = FileSystem.get(configuration);
  Path configPath = new Path(configuration.get(ROLLUP_PHASE3_CONFIG_PATH.toString()));
  try {
    StarTreeConfig starTreeConfig = StarTreeConfig.decode(fileSystem.open(configPath));
    config = RollupPhaseThreeConfig.fromStarTreeConfig(starTreeConfig);
    dimensionNames = config.getDimensionNames();
    dimensionNameToIndexMapping = new HashMap<String, Integer>();

    for (int i = 0; i < dimensionNames.size(); i++) {
      dimensionNameToIndexMapping.put(dimensionNames.get(i), i);
    }
    metricNames = config.getMetricNames();
    metricTypes = config.getMetricTypes();
    metricSchema = new MetricSchema(config.getMetricNames(), metricTypes);
  } catch (Exception e) {
    throw new IOException(e);
  }
}
開發者ID:Hanmourang，項目名稱:Pinot，代碼行數:24，代碼來源:RollupPhaseThreeJob.java

示例13: setup

import org.apache.hadoop.mapreduce.lib.output.MultipleOutputs; //導入依賴的package包/類
@Override
public void setup(Context context) throws IOException, InterruptedException {
  LOGGER.info("RollupPhaseOneJob.RollupPhaseOneMapper.setup()");
  mos = new MultipleOutputs<BytesWritable, BytesWritable>(context);
  Configuration configuration = context.getConfiguration();
  FileSystem fileSystem = FileSystem.get(configuration);
  Path configPath = new Path(configuration.get(ROLLUP_PHASE1_CONFIG_PATH
      .toString()));
  try {
    StarTreeConfig starTreeConfig = StarTreeConfig.decode(fileSystem.open(configPath));
    config = RollupPhaseOneConfig.fromStarTreeConfig(starTreeConfig);
    dimensionNames = config.getDimensionNames();
    metricTypes = config.getMetricTypes();
    metricSchema = new MetricSchema(config.getMetricNames(), metricTypes);
    String className = config.getThresholdFuncClassName();
    Map<String,String> params = config.getThresholdFuncParams();
    Constructor<?> constructor = Class.forName(className).getConstructor(Map.class);
    thresholdFunc = (RollupThresholdFunction) constructor.newInstance(params);

  } catch (Exception e) {
    throw new IOException(e);
  }
}
開發者ID:Hanmourang，項目名稱:Pinot，代碼行數:24，代碼來源:RollupPhaseOneJob.java

示例14: setup

import org.apache.hadoop.mapreduce.lib.output.MultipleOutputs; //導入依賴的package包/類
@Override
public void setup(Context context) throws IOException, InterruptedException {
  LOGGER.info("RollupPhaseOneJob.RollupPhaseOneMapper.setup()");
  mos = new MultipleOutputs<BytesWritable, BytesWritable>(context);
  Configuration configuration = context.getConfiguration();
  FileSystem fileSystem = FileSystem.get(configuration);
  Path configPath = new Path(configuration.get(ROLLUP_PHASE4_CONFIG_PATH.toString()));
  try {
    StarTreeConfig starTreeConfig = StarTreeConfig.decode(fileSystem.open(configPath));
    config = RollupPhaseFourConfig.fromStarTreeConfig(starTreeConfig);
    dimensionNames = config.getDimensionNames();
    dimensionNameToIndexMapping = new HashMap<String, Integer>();

    for (int i = 0; i < dimensionNames.size(); i++) {
      dimensionNameToIndexMapping.put(dimensionNames.get(i), i);
    }
    metricNames = config.getMetricNames();
    metricTypes = config.getMetricTypes();
    metricSchema = new MetricSchema(config.getMetricNames(), metricTypes);
    rollupOrder = config.getRollupOrder();
  } catch (Exception e) {
    throw new IOException(e);
  }
}
開發者ID:Hanmourang，項目名稱:Pinot，代碼行數:25，代碼來源:RollupPhaseFourJob.java

示例15: doSetup

import org.apache.hadoop.mapreduce.lib.output.MultipleOutputs; //導入依賴的package包/類
@Override
protected void doSetup(Context context) throws IOException {
    super.bindCurrentConfiguration(context.getConfiguration());
    mos = new MultipleOutputs(context);

    String cubeName = context.getConfiguration().get(BatchConstants.CFG_CUBE_NAME);
    String segmentID = context.getConfiguration().get(BatchConstants.CFG_CUBE_SEGMENT_ID);

    KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata();

    CubeManager cubeManager = CubeManager.getInstance(config);
    CubeInstance cube = cubeManager.getCube(cubeName);
    CubeSegment optSegment = cube.getSegmentById(segmentID);
    CubeSegment originalSegment = cube.getOriginalSegmentToOptimize(optSegment);

    rowKeySplitter = new RowKeySplitter(originalSegment, 65, 255);
    baseCuboid = cube.getCuboidScheduler().getBaseCuboidId();

    recommendCuboids = cube.getCuboidsRecommend();
    Preconditions.checkNotNull(recommendCuboids, "The recommend cuboid map could not be null");
}
開發者ID:apache，項目名稱:kylin，代碼行數:22，代碼來源:FilterRecommendCuboidDataMapper.java


注：本文中的org.apache.hadoop.mapreduce.lib.output.MultipleOutputs類示例由純淨天空整理自Github/MSDocs等開源代碼及文檔管理平台，相關代碼片段篩選自各路編程大神貢獻的開源項目，源碼版權歸原作者所有，傳播和使用請參考對應項目的License；未經允許，請勿轉載。