當前位置: 首頁>>代碼示例>>Java>>正文

Java FileOutputFormat.setOutputCompressorClass方法代碼示例

本文整理匯總了Java中org.apache.hadoop.mapred.FileOutputFormat.setOutputCompressorClass方法的典型用法代碼示例。如果您正苦於以下問題：Java FileOutputFormat.setOutputCompressorClass方法的具體用法？Java FileOutputFormat.setOutputCompressorClass怎麽用？Java FileOutputFormat.setOutputCompressorClass使用的例子？那麽, 這裏精選的方法代碼示例或許可以為您提供幫助。您也可以進一步了解該方法所在類org.apache.hadoop.mapred.FileOutputFormat的用法示例。


在下文中一共展示了FileOutputFormat.setOutputCompressorClass方法的4個代碼示例，這些例子默認根據受歡迎程度排序。您可以為喜歡或者感覺有用的代碼點讚，您的評價將有助於係統推薦出更棒的Java代碼示例。

示例1: createPageRankNodesDirectly

import org.apache.hadoop.mapred.FileOutputFormat; //導入方法依賴的package包/類
private void createPageRankNodesDirectly() throws IOException {

		log.info("Creating PageRank nodes...", null);

		Path fout = new Path(options.getResultPath(), VERTICALS_DIR_NAME);
		
		JobConf job = new JobConf(PagerankData.class);
		String jobname = "Create pagerank nodes";

		job.setJobName(jobname);
		setPageRankNodesOptions(job);

		job.setOutputKeyClass(LongWritable.class);
		job.setOutputValueClass(Text.class);
		
		FileInputFormat.setInputPaths(job, dummy.getPath());
		job.setInputFormat(NLineInputFormat.class);
		
		if (balance) {
			/***
			 * Balance the output order of nodes, to prevent the running
			 * of pagerank bench from potential data skew
			 */
			job.setMapOutputKeyClass(LongWritable.class);
			job.setMapOutputValueClass(NullWritable.class);
			
			job.setMapperClass(BalancedLinkNodesMapper.class);
			job.setReducerClass(BalancedLinkNodesReducer.class);
//			job.setPartitionerClass(ModulusPartitioner.class);

			if (options.getNumReds() > 0) {
				job.setNumReduceTasks(options.getNumReds());
			} else {
				job.setNumReduceTasks(Utils.getMaxNumReds());
			}
		} else {
			job.setMapOutputKeyClass(Text.class);
			job.setMapperClass(DummyToNodesMapper.class);
			job.setNumReduceTasks(0);
		}

		if (options.isSequenceOut()) {
			job.setOutputFormat(SequenceFileOutputFormat.class);
		} else {
			job.setOutputFormat(TextOutputFormat.class);
		}
		
		if (null != options.getCodecClass()) {
			job.set("mapred.output.compression.type","BLOCK");
			job.set("mapreduce.output.fileoutputformat.compress.type","BLOCK");
			FileOutputFormat.setCompressOutput(job, true);
			FileOutputFormat.setOutputCompressorClass(job, options.getCodecClass());
		}
		
		FileOutputFormat.setOutputPath(job, fout);

		log.info("Running Job: " +jobname);
		log.info("Dummy file " + dummy.getPath() + " as input");
		log.info("Vertices file " + fout + " as output");
		JobClient.runJob(job);
		log.info("Finished Running Job: " + jobname);
	}
開發者ID:thrill，項目名稱:fst-bench，代碼行數:63，代碼來源:PagerankData.java

示例2: createPageRankLinksDirectly

import org.apache.hadoop.mapred.FileOutputFormat; //導入方法依賴的package包/類
private void createPageRankLinksDirectly() throws IOException, URISyntaxException {

		log.info("Creating PageRank links", null);

		JobConf job = new JobConf(PagerankData.class);
		String jobname = "Create pagerank links";

		Path fout = new Path(options.getResultPath(), EDGES_DIR_NAME);

		job.setJobName(jobname);
		setPageRankLinksOptions(job);

		job.setOutputKeyClass(LongWritable.class);
		job.setOutputValueClass(Text.class);
//		job.setMapOutputKeyClass(LongWritable.class);
//		job.setMapOutputValueClass(Text.class);

		job.setNumReduceTasks(0);
		
		FileInputFormat.setInputPaths(job, dummy.getPath());
		job.setInputFormat(NLineInputFormat.class);

		job.setMapperClass(DummyToPageRankLinksMapper.class);

		if (options.isSequenceOut()) {
			job.setOutputFormat(SequenceFileOutputFormat.class);
		} else {
			job.setOutputFormat(TextOutputFormat.class);
		}
		
		if (null != options.getCodecClass()) {
			job.set("mapred.output.compression.type","BLOCK");
			job.set("mapreduce.output.fileoutputformat.compress.type", "BLOCK");
			FileOutputFormat.setCompressOutput(job, true);
			FileOutputFormat.setOutputCompressorClass(job, options.getCodecClass());
		}
		
		FileOutputFormat.setOutputPath(job, fout);
		
		log.info("Running Job: " +jobname);
		log.info("Dummy file " + dummy.getPath() + " as input");
		log.info("Edges file " + fout + " as output");
		JobClient.runJob(job);
		log.info("Finished Running Job: " + jobname);
	}
開發者ID:thrill，項目名稱:fst-bench，代碼行數:46，代碼來源:PagerankData.java

示例3: createRankingsTableDirectly

import org.apache.hadoop.mapred.FileOutputFormat; //導入方法依賴的package包/類
private void createRankingsTableDirectly() throws IOException, URISyntaxException {

		log.info("Creating table rankings...");

		Path fout = new Path(options.getResultPath(), RANKINGS);

		JobConf job = new JobConf(HiveData.class);
		String jobname = "Create rankings";

		/** TODO: change another more effective way as this operation may cause
		 *  about 2 min delay (originally ~15min in total)
		 */
		setRankingsOptions(job);
		job.setJobName(jobname);
		job.set("mapred.reduce.slowstart.completed.maps", "0.3");
		job.set("mapreduce.job.reduce.slowstart.completedmaps", "0.3");

		job.setOutputKeyClass(LongWritable.class);
		job.setOutputValueClass(Text.class);

		job.setMapOutputKeyClass(LongWritable.class);
		job.setMapOutputValueClass(JoinBytesInt.class);

		job.setJarByClass(DummyToRankingsMapper.class);
		job.setJarByClass(JoinBytesIntCombiner.class);
		job.setJarByClass(GenerateRankingsReducer.class);
		
		job.setMapperClass(DummyToRankingsMapper.class);
		job.setCombinerClass(JoinBytesIntCombiner.class);
		job.setReducerClass(GenerateRankingsReducer.class);

		if (options.getNumReds() > 0) {
			job.setNumReduceTasks(options.getNumReds());
		} else {
			job.setNumReduceTasks(Utils.getMaxNumReds());
		}

		job.setInputFormat(NLineInputFormat.class);
		FileInputFormat.setInputPaths(job, dummy.getPath());

		job.set("mapred.map.output.compression.type", "BLOCK");
	 	job.set("mapreduce.output.fileoutputformat.compress.type","BLOCK");	
		MapFileOutputFormat.setCompressOutput(job, true);
//		MapFileOutputFormat.setOutputCompressorClass(job, org.apache.hadoop.io.compress.LzoCodec.class);
		MapFileOutputFormat.setOutputCompressorClass(job, org.apache.hadoop.io.compress.DefaultCodec.class);

		if (options.isSequenceOut()) {
			job.setOutputFormat(SequenceFileOutputFormat.class);
		} else {
			job.setOutputFormat(TextOutputFormat.class);
		}
		
		if (null != options.getCodecClass()) {
			job.set("mapred.output.compression.type","BLOCK");
		 	job.set("mapreduce.output.fileoutputformat.compress.type","BLOCK");	
			FileOutputFormat.setCompressOutput(job, true);
			FileOutputFormat.setOutputCompressorClass(job, options.getCodecClass());
		}

		FileOutputFormat.setOutputPath(job, fout);

		log.info("Running Job: " +jobname);
		log.info("Pages file " + dummy.getPath() + " as input");
		log.info("Rankings file " + fout + " as output");
		JobClient.runJob(job);
		log.info("Finished Running Job: " + jobname);
	}
開發者ID:thrill，項目名稱:fst-bench，代碼行數:68，代碼來源:HiveData.java

示例4: createUserVisitsTableDirectly

import org.apache.hadoop.mapred.FileOutputFormat; //導入方法依賴的package包/類
private void createUserVisitsTableDirectly() throws IOException, URISyntaxException {

		log.info("Creating user visits...");

		Path rankings = new Path(options.getResultPath(), RANKINGS);
		Path fout = new Path(options.getResultPath(), USERVISITS);

		JobConf job = new JobConf(HiveData.class);
		String jobname = "Create uservisits";
		job.setJobName(jobname);
		setVisitsOptions(job);
		
		/***
		 * Set distributed cache file for table generation,
		 * cache files include:
		 * 1. user agents
		 * 2. country code and language code
		 * 3. search keys
		 */

		Path uagentPath = new Path(options.getWorkPath(), uagentf);
		DistributedCache.addCacheFile(uagentPath.toUri(), job);
		
		Path countryPath = new Path(options.getWorkPath(), countryf);
		DistributedCache.addCacheFile(countryPath.toUri(), job);

		Path searchkeyPath = new Path(options.getWorkPath(), searchkeyf);
		DistributedCache.addCacheFile(searchkeyPath.toUri(), job);

		job.setOutputKeyClass(LongWritable.class);
		job.setOutputValueClass(Text.class);

		job.setMapOutputKeyClass(LongWritable.class);
		job.setMapOutputValueClass(JoinBytesInt.class);

		MultipleInputs.addInputPath(job, dummy.getPath(),
				NLineInputFormat.class, DummyToAccessNoMapper.class);

		if (options.isSequenceOut()) {
			MultipleInputs.addInputPath(job, rankings,
					SequenceFileInputFormat.class, SequenceRankingsToUrlsMapper.class);
		} else {
			MultipleInputs.addInputPath(job, rankings,
					TextInputFormat.class, TextRankingsToUrlsMapper.class);
		}

		job.setCombinerClass(JoinBytesIntCombiner.class);
		job.setReducerClass(CreateUserVisitsReducer.class);
		
		if (options.getNumReds() > 0) {
			job.setNumReduceTasks(options.getNumReds());
		} else {
			job.setNumReduceTasks(Utils.getMaxNumReds());
		}

//		job.setNumReduceTasks(options.slots/2);

		if (options.isSequenceOut()) {
			job.setOutputFormat(SequenceFileOutputFormat.class);
		} else {
			job.setOutputFormat(TextOutputFormat.class);
		}
		
		if (null != options.getCodecClass()) {
			job.set("mapred.output.compression.type","BLOCK");
                        job.set("mapreduce.output.fileoutputformat.compress.type","BLOCK");
			FileOutputFormat.setCompressOutput(job, true);
			FileOutputFormat.setOutputCompressorClass(job, options.getCodecClass());
		}
		
		FileOutputFormat.setOutputPath(job, fout);
		
		log.info("Running Job: " +jobname);
		log.info("Dummy file " + dummy.getPath() + " as input");
		log.info("Rankings file " + rankings + " as input");
		log.info("Ouput file " + fout);
		JobClient.runJob(job);
		log.info("Finished Running Job: " + jobname);
	}
開發者ID:thrill，項目名稱:fst-bench，代碼行數:80，代碼來源:HiveData.java


注：本文中的org.apache.hadoop.mapred.FileOutputFormat.setOutputCompressorClass方法示例由純淨天空整理自Github/MSDocs等開源代碼及文檔管理平台，相關代碼片段篩選自各路編程大神貢獻的開源項目，源碼版權歸原作者所有，傳播和使用請參考對應項目的License；未經允許，請勿轉載。