當前位置: 首頁>>代碼示例>>Java>>正文

Java InvalidJobConfException類代碼示例

本文整理匯總了Java中org.apache.hadoop.mapred.InvalidJobConfException類的典型用法代碼示例。如果您正苦於以下問題：Java InvalidJobConfException類的具體用法？Java InvalidJobConfException怎麽用？Java InvalidJobConfException使用的例子？那麽, 這裏精選的類代碼示例或許可以為您提供幫助。


InvalidJobConfException類屬於org.apache.hadoop.mapred包，在下文中一共展示了InvalidJobConfException類的15個代碼示例，這些例子默認根據受歡迎程度排序。您可以為喜歡或者感覺有用的代碼點讚，您的評價將有助於係統推薦出更棒的Java代碼示例。

示例1: checkOutputSpecs

import org.apache.hadoop.mapred.InvalidJobConfException; //導入依賴的package包/類
public void checkOutputSpecs(JobContext job
                             ) throws FileAlreadyExistsException, IOException{
  // Ensure that the output directory is set and not already there
  Path outDir = getOutputPath(job);
  if (outDir == null) {
    throw new InvalidJobConfException("Output directory not set.");
  }

  // get delegation token for outDir's file system
  TokenCache.obtainTokensForNamenodes(job.getCredentials(),
      new Path[] { outDir }, job.getConfiguration());

  if (outDir.getFileSystem(job.getConfiguration()).exists(outDir)) {
    throw new FileAlreadyExistsException("Output directory " + outDir + 
                                         " already exists");
  }
}
開發者ID:naver，項目名稱:hadoop，代碼行數:18，代碼來源:FileOutputFormat.java

示例2: checkOutputSpecs

import org.apache.hadoop.mapred.InvalidJobConfException; //導入依賴的package包/類
public void checkOutputSpecs(JobContext job
                             ) throws FileAlreadyExistsException, IOException{
  // Ensure that the output directory is set and not already there
  Path outDir = getOutputPath(job);
  if (outDir == null) {
    throw new InvalidJobConfException("Output directory not set.");
  }
  
  // get delegation token for outDir's file system
  TokenCache.obtainTokensForNamenodes(job.getCredentials(), 
                                      new Path[] {outDir}, 
                                      job.getConfiguration());

  if (outDir.getFileSystem(job.getConfiguration()).exists(outDir)) {
    throw new FileAlreadyExistsException("Output directory " + outDir + 
                                         " already exists");
  }
}
開發者ID:Nextzero，項目名稱:hadoop-2.6.0-cdh5.4.3，代碼行數:19，代碼來源:FileOutputFormat.java

示例3: checkOutputSpecs

import org.apache.hadoop.mapred.InvalidJobConfException; //導入依賴的package包/類
/**
 * Overridden to avoid throwing an exception if the specified directory
 * for export already exists.
 */
@Override
public void checkOutputSpecs(JobContext job) throws FileAlreadyExistsException, IOException {
    Path outDir = getOutputPath(job);
    if(outDir == null) {
        throw new InvalidJobConfException("Output directory not set.");
    } else {
        TokenCache.obtainTokensForNamenodes(job.getCredentials(), new Path[]{outDir}, job.getConfiguration());
        /*
        if(outDir.getFileSystem(job.getConfiguration()).exists(outDir)) {
            System.out.println("Output dir already exists, no problem");
            throw new FileAlreadyExistsException("Output directory " + outDir + " already exists");
        }
        */
    }
}
開發者ID:splicemachine，項目名稱:spliceengine，代碼行數:20，代碼來源:SparkDataSet.java

示例4: checkOutputSpecs

import org.apache.hadoop.mapred.InvalidJobConfException; //導入依賴的package包/類
@Override
public void checkOutputSpecs(FileSystem ignored, JobConf job) throws FileAlreadyExistsException, InvalidJobConfException, IOException {
    // Ensure that the output directory is set and not already there
    Path outDir = getOutputPath(job);
    if (outDir == null && job.getNumReduceTasks() != 0) {
        throw new InvalidJobConfException("Output directory not set in JobConf.");
    }
    if (outDir != null) {
        FileSystem fs = outDir.getFileSystem(job);
        // normalize the output directory
        outDir = fs.makeQualified(outDir);
        setOutputPath(job, outDir);

        // get delegation token for the outDir's file system
        TokenCache.obtainTokensForNamenodes(job.getCredentials(), new Path[]{outDir}, job);
        String jobUuid = job.get("zephyr.job.uuid");
        if (jobUuid == null)
            throw new InvalidJobConfException("This output format REQUIRES the value zephyr.job.uuid to be specified in the job configuration!");
        // // check its existence
        // if (fs.exists(outDir)) {
        // throw new FileAlreadyExistsException("Output directory " + outDir
        // + " already exists");
        // }
    }
}
開發者ID:Sotera，項目名稱:zephyr，代碼行數:26，代碼來源:ZephyrOutputFormat.java

示例5: checkOutputSpecs

import org.apache.hadoop.mapred.InvalidJobConfException; //導入依賴的package包/類
@Override 
public void checkOutputSpecs(JobContext job) throws IOException {
  super.checkOutputSpecs(job);
  if (getCompressOutput(job) && 
      getOutputCompressionType(job) == CompressionType.RECORD ) {
    throw new InvalidJobConfException("SequenceFileAsBinaryOutputFormat "
      + "doesn't support Record Compression" );
  }
}
開發者ID:naver，項目名稱:hadoop，代碼行數:10，代碼來源:SequenceFileAsBinaryOutputFormat.java

示例6: checkOutputSpecs

import org.apache.hadoop.mapred.InvalidJobConfException; //導入依賴的package包/類
@Override
public void checkOutputSpecs(JobContext job
                            ) throws InvalidJobConfException, IOException {
  // Ensure that the output directory is set
  Path outDir = getOutputPath(job);
  if (outDir == null) {
    throw new InvalidJobConfException("Output directory not set in JobConf.");
  }

  final Configuration jobConf = job.getConfiguration();

  // get delegation token for outDir's file system
  TokenCache.obtainTokensForNamenodes(job.getCredentials(),
      new Path[] { outDir }, jobConf);

  final FileSystem fs = outDir.getFileSystem(jobConf);

  if (fs.exists(outDir)) {
    // existing output dir is considered empty iff its only content is the
    // partition file.
    //
    final FileStatus[] outDirKids = fs.listStatus(outDir);
    boolean empty = false;
    if (outDirKids != null && outDirKids.length == 1) {
      final FileStatus st = outDirKids[0];
      final String fname = st.getPath().getName();
      empty =
        !st.isDirectory() && TeraInputFormat.PARTITION_FILENAME.equals(fname);
    }
    if (TeraSort.getUseSimplePartitioner(job) || !empty) {
      throw new FileAlreadyExistsException("Output directory " + outDir
          + " already exists");
    }
  }
}
開發者ID:naver，項目名稱:hadoop，代碼行數:36，代碼來源:TeraOutputFormat.java

示例7: checkOutputSpecs

import org.apache.hadoop.mapred.InvalidJobConfException; //導入依賴的package包/類
@Override
public void checkOutputSpecs(FileSystem ignored, JobConf job)
throws FileAlreadyExistsException, InvalidJobConfException, IOException {
  String tableName = job.get(OUTPUT_TABLE);
  if (tableName == null) {
    throw new IOException("Must specify table name");
  }
}
開發者ID:fengchen8086，項目名稱:ditb，代碼行數:9，代碼來源:TableOutputFormat.java

示例8: testSetupDistributedCacheConflicts

import org.apache.hadoop.mapred.InvalidJobConfException; //導入依賴的package包/類
@SuppressWarnings("deprecation")
@Test(timeout = 120000, expected = InvalidJobConfException.class)
public void testSetupDistributedCacheConflicts() throws Exception {
  Configuration conf = new Configuration();
  conf.setClass("fs.mockfs.impl", MockFileSystem.class, FileSystem.class);
  
  URI mockUri = URI.create("mockfs://mock/");
  FileSystem mockFs = ((FilterFileSystem)FileSystem.get(mockUri, conf))
      .getRawFileSystem();
  
  URI archive = new URI("mockfs://mock/tmp/something.zip#something");
  Path archivePath = new Path(archive);
  URI file = new URI("mockfs://mock/tmp/something.txt#something");
  Path filePath = new Path(file);
  
  when(mockFs.resolvePath(archivePath)).thenReturn(archivePath);
  when(mockFs.resolvePath(filePath)).thenReturn(filePath);
  
  DistributedCache.addCacheArchive(archive, conf);
  conf.set(MRJobConfig.CACHE_ARCHIVES_TIMESTAMPS, "10");
  conf.set(MRJobConfig.CACHE_ARCHIVES_SIZES, "10");
  conf.set(MRJobConfig.CACHE_ARCHIVES_VISIBILITIES, "true");
  DistributedCache.addCacheFile(file, conf);
  conf.set(MRJobConfig.CACHE_FILE_TIMESTAMPS, "11");
  conf.set(MRJobConfig.CACHE_FILES_SIZES, "11");
  conf.set(MRJobConfig.CACHE_FILE_VISIBILITIES, "true");
  Map<String, LocalResource> localResources = 
    new HashMap<String, LocalResource>();
  MRApps.setupDistributedCache(conf, localResources);
}
開發者ID:aliyun-beta，項目名稱:aliyun-oss-hadoop-fs，代碼行數:31，代碼來源:TestMRApps.java

示例9: testSetupDistributedCacheConflictsFiles

import org.apache.hadoop.mapred.InvalidJobConfException; //導入依賴的package包/類
@SuppressWarnings("deprecation")
@Test(timeout = 120000, expected = InvalidJobConfException.class)
public void testSetupDistributedCacheConflictsFiles() throws Exception {
  Configuration conf = new Configuration();
  conf.setClass("fs.mockfs.impl", MockFileSystem.class, FileSystem.class);
  
  URI mockUri = URI.create("mockfs://mock/");
  FileSystem mockFs = ((FilterFileSystem)FileSystem.get(mockUri, conf))
      .getRawFileSystem();
  
  URI file = new URI("mockfs://mock/tmp/something.zip#something");
  Path filePath = new Path(file);
  URI file2 = new URI("mockfs://mock/tmp/something.txt#something");
  Path file2Path = new Path(file2);
  
  when(mockFs.resolvePath(filePath)).thenReturn(filePath);
  when(mockFs.resolvePath(file2Path)).thenReturn(file2Path);
  
  DistributedCache.addCacheFile(file, conf);
  DistributedCache.addCacheFile(file2, conf);
  conf.set(MRJobConfig.CACHE_FILE_TIMESTAMPS, "10,11");
  conf.set(MRJobConfig.CACHE_FILES_SIZES, "10,11");
  conf.set(MRJobConfig.CACHE_FILE_VISIBILITIES, "true,true");
  Map<String, LocalResource> localResources = 
    new HashMap<String, LocalResource>();
  MRApps.setupDistributedCache(conf, localResources);
}
開發者ID:aliyun-beta，項目名稱:aliyun-oss-hadoop-fs，代碼行數:28，代碼來源:TestMRApps.java

示例10: checkOutputSpecs

import org.apache.hadoop.mapred.InvalidJobConfException; //導入依賴的package包/類
@Override
public void checkOutputSpecs(JobContext job
                            ) throws InvalidJobConfException, IOException {
  // Ensure that the output directory is set
  Path outDir = getOutputPath(job);
  if (outDir == null) {
    throw new InvalidJobConfException("Output directory not set in JobConf.");
  }

  // get delegation token for outDir's file system
  TokenCache.obtainTokensForNamenodes(job.getCredentials(),
      new Path[] { outDir }, job.getConfiguration());
}
開發者ID:thrill，項目名稱:fst-bench，代碼行數:14，代碼來源:TeraOutputFormat.java

示例11: checkOutputSpecs

import org.apache.hadoop.mapred.InvalidJobConfException; //導入依賴的package包/類
public void checkOutputSpecs(FileSystem fs, JobConf job) throws IOException {
	Path out = FileOutputFormat.getOutputPath(job);
	if ((out == null) && (job.getNumReduceTasks() != 0)) {
		throw new InvalidJobConfException(
				"Output directory not set in JobConf.");
	}
	if (fs == null) {
		fs = out.getFileSystem(job);
	}

	if (fs.exists(new Path(out, CrawlDatum.PARSE_DIR_NAME)))
		throw new IOException("Segment already parsed!");
	
	
}
開發者ID:thrill，項目名稱:fst-bench，代碼行數:16，代碼來源:NutchOutputFormat.java

示例12: checkOutputSpecs

import org.apache.hadoop.mapred.InvalidJobConfException; //導入依賴的package包/類
public void checkOutputSpecs(FileSystem fs, JobConf job) throws IOException {
  Path out = FileOutputFormat.getOutputPath(job);
  if ((out == null) && (job.getNumReduceTasks() != 0)) {
    throw new InvalidJobConfException("Output directory not set in JobConf.");
  }
  if (fs == null) {
    fs = out.getFileSystem(job);
  }
  if (fs.exists(new Path(out, CrawlDatum.FETCH_DIR_NAME)))
    throw new IOException("Segment already fetched!");
}
開發者ID:jorcox，項目名稱:GeoCrawler，代碼行數:12，代碼來源:FetcherOutputFormat.java

示例13: MapRedGfxdRecordWriter

import org.apache.hadoop.mapred.InvalidJobConfException; //導入依賴的package包/類
public MapRedGfxdRecordWriter(Configuration conf) throws IOException {
  this.tableName = conf.get(OUTPUT_TABLE);
  try {
    this.batchExecutor = util.new RowCommandBatchExecutor(getDriver(conf),
        conf.get(OUTPUT_URL), conf.getInt(OUTPUT_BATCH_SIZE,
            OUTPUT_BATCH_SIZE_DEFAULT));
  } catch (ClassNotFoundException e) {
    logger.error("Gemfirexd client classes are missing from the classpath", e);
    throw new InvalidJobConfException(e);
  }
}
開發者ID:gemxd，項目名稱:gemfirexd-oss，代碼行數:12，代碼來源:RowOutputFormat.java

示例14: validateConfiguration

import org.apache.hadoop.mapred.InvalidJobConfException; //導入依賴的package包/類
/**
 * Validates correctness and completeness of job's output configuration. Job
 * configuration must contain url, table and schema name
 * 
 * @param conf
 *          job conf
 * @throws InvalidJobConfException
 */
protected static void validateConfiguration(Configuration conf)
    throws InvalidJobConfException {
  // User must configure the output region name.
  String url = conf.get(OUTPUT_URL);
  if (url == null || url.trim().isEmpty()) {
    throw new InvalidJobConfException("Output URL not configured.");
  }

  String table = conf.get(OUTPUT_TABLE);
  if (table == null || table.trim().isEmpty()) {
    throw new InvalidJobConfException("Output table name not provided.");
  }
}
開發者ID:gemxd，項目名稱:gemfirexd-oss，代碼行數:22，代碼來源:RowOutputFormat.java

示例15: GfxdRecordWriter

import org.apache.hadoop.mapred.InvalidJobConfException; //導入依賴的package包/類
public GfxdRecordWriter(Configuration conf) throws IOException {
  this.tableName = conf.get(OUTPUT_TABLE);
  try {
    this.batchExecutor = util.new RowCommandBatchExecutor(getDriver(conf),
        conf.get(OUTPUT_URL), conf.getInt(OUTPUT_BATCH_SIZE,
            OUTPUT_BATCH_SIZE_DEFAULT));
  } catch (ClassNotFoundException e) {
    logger.error("Gemfirexd client classes are missing from the classpath", e);
    throw new InvalidJobConfException(e);
  }
}
開發者ID:gemxd，項目名稱:gemfirexd-oss，代碼行數:12，代碼來源:RowOutputFormat.java


注：本文中的org.apache.hadoop.mapred.InvalidJobConfException類示例由純淨天空整理自Github/MSDocs等開源代碼及文檔管理平台，相關代碼片段篩選自各路編程大神貢獻的開源項目，源碼版權歸原作者所有，傳播和使用請參考對應項目的License；未經允許，請勿轉載。