Java Configuration类代码示例

import org.apache.hadoop.conf.Configuration; //导入依赖的package包/类
@Override
/** {@inheritDoc} */
protected RecordReader<LongWritable, T> createDBRecordReader(
    DBInputSplit split, Configuration conf) throws IOException {

  DBConfiguration dbConf = getDBConf();
  Class<T> inputClass = (Class<T>) (dbConf.getInputClass());
  String dbProductName = getDBProductName();
  LOG.debug("Creating db record reader for db product: " + dbProductName);

  try {
    return new SQLServerDBRecordReader<T>(split, inputClass,
        conf, getConnection(), dbConf, dbConf.getInputConditions(),
        dbConf.getInputFieldNames(), dbConf.getInputTableName(),
        dbProductName);
  } catch (SQLException ex) {
    throw new IOException(ex);
  }
}
 

import org.apache.hadoop.conf.Configuration; //导入依赖的package包/类
protected void open(Path dstPath, CompressionCodec codeC,
    CompressionType compType, Configuration conf, FileSystem hdfs)
        throws IOException {
  if(useRawLocalFileSystem) {
    if(hdfs instanceof LocalFileSystem) {
      hdfs = ((LocalFileSystem)hdfs).getRaw();
    } else {
      logger.warn("useRawLocalFileSystem is set to true but file system " +
          "is not of type LocalFileSystem: " + hdfs.getClass().getName());
    }
  }
  if (conf.getBoolean("hdfs.append.support", false) == true && hdfs.isFile
          (dstPath)) {
    outStream = hdfs.append(dstPath);
  } else {
    outStream = hdfs.create(dstPath);
  }
  writer = SequenceFile.createWriter(conf, outStream,
      serializer.getKeyClass(), serializer.getValueClass(), compType, codeC);

  registerCurrentStream(outStream, hdfs, dstPath);
}
 

import org.apache.hadoop.conf.Configuration; //导入依赖的package包/类
@Override
public void init(Configuration configuration, RMProxy<T> rmProxy,
                  Class<T> protocol) {
  this.rmProxy = rmProxy;
  this.protocol = protocol;
  this.rmProxy.checkAllowedProtocols(this.protocol);
  this.conf = new YarnConfiguration(configuration);
  Collection<String> rmIds = HAUtil.getRMHAIds(conf);
  this.rmServiceIds = rmIds.toArray(new String[rmIds.size()]);
  conf.set(YarnConfiguration.RM_HA_ID, rmServiceIds[currentProxyIndex]);

  conf.setInt(CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY,
      conf.getInt(YarnConfiguration.CLIENT_FAILOVER_RETRIES,
          YarnConfiguration.DEFAULT_CLIENT_FAILOVER_RETRIES));

  conf.setInt(CommonConfigurationKeysPublic.
      IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SOCKET_TIMEOUTS_KEY,
      conf.getInt(YarnConfiguration.CLIENT_FAILOVER_RETRIES_ON_SOCKET_TIMEOUTS,
          YarnConfiguration.DEFAULT_CLIENT_FAILOVER_RETRIES_ON_SOCKET_TIMEOUTS));
}
 

import org.apache.hadoop.conf.Configuration; //导入依赖的package包/类
/**
 * 定义每条数据的输出格式，输入数据是由reduce任务每次执行write方法输出的数据
 */
@Override
public RecordWriter<BaseDimension, BaseStatsValueWritable> getRecordWriter(TaskAttemptContext context)
		throws IOException, InterruptedException {
	Configuration conf = context.getConfiguration();
	Connection conn = null;
	IDimensionConverter converter = new DimensionConverterImpl();
	try {
		conn = JdbcManager.getConnection(conf, GlobalConstants.WAREHOUSE_OF_REPORT);
		conn.setAutoCommit(false);
	} catch (SQLException e) {
		logger.error("获取数据库连接失败", e);
		throw new IOException("获取数据库连接失败", e);
	}
	return new TransformerRecordWriter(conn, conf, converter);
}
 

import org.apache.hadoop.conf.Configuration; //导入依赖的package包/类
/**
 * See {@link DirectoryScanner}
 */
private synchronized void initDirectoryScanner(Configuration conf) {
  if (directoryScanner != null) {
    return;
  }
  String reason = null;
  if (conf.getInt(DFS_DATANODE_DIRECTORYSCAN_INTERVAL_KEY, 
                  DFS_DATANODE_DIRECTORYSCAN_INTERVAL_DEFAULT) < 0) {
    reason = "verification is turned off by configuration";
  } else if ("SimulatedFSDataset".equals(data.getClass().getSimpleName())) {
    reason = "verifcation is not supported by SimulatedFSDataset";
  } 
  if (reason == null) {
    directoryScanner = new DirectoryScanner(this, data, conf);
    directoryScanner.start();
  } else {
    LOG.info("Periodic Directory Tree Verification scan is disabled because " +
                 reason);
  }
}
 

import org.apache.hadoop.conf.Configuration; //导入依赖的package包/类
@Test
public void testFailoverToNonExistantServiceFails() throws Exception {
  DummyHAService svc1 = new DummyHAService(HAServiceState.ACTIVE, svc1Addr);
  DummyHAService svc2 = spy(new DummyHAService(null, svc2Addr));
  Mockito.doThrow(new IOException("Failed to connect"))
    .when(svc2).getProxy(Mockito.<Configuration>any(),
        Mockito.anyInt());
  svc1.fencer = svc2.fencer = setupFencer(AlwaysSucceedFencer.class.getName());

  try {
    doFailover(svc1, svc2, false, false);
    fail("Failed over to a non-existant standby");
  } catch (FailoverFailedException ffe) {
    // Expected
  }

  assertEquals(HAServiceState.ACTIVE, svc1.state);
}
 

import org.apache.hadoop.conf.Configuration; //导入依赖的package包/类
@SuppressWarnings("deprecation")
private void testDelegationTokenWithFS(Class fileSystemClass)
  throws Exception {
  createHttpFSServer();
  Configuration conf = new Configuration();
  conf.set("fs.webhdfs.impl", fileSystemClass.getName());
  conf.set("fs.hdfs.impl.disable.cache", "true");
  URI uri = new URI( "webhdfs://" +
                     TestJettyHelper.getJettyURL().toURI().getAuthority());
  FileSystem fs = FileSystem.get(uri, conf);
  Token<?> tokens[] = fs.addDelegationTokens("foo", null);
  fs.close();
  Assert.assertEquals(1, tokens.length);
  fs = FileSystem.get(uri, conf);
  ((DelegationTokenRenewer.Renewable) fs).setDelegationToken(tokens[0]);
  fs.listStatus(new Path("/"));
  fs.close();
}
 

import org.apache.hadoop.conf.Configuration; //导入依赖的package包/类
/**
 * Initialize {@link DFSConfigKeys#DFS_NAMENODE_NAME_DIR_KEY} and 
 * {@link DFSConfigKeys#DFS_DATANODE_DATA_DIR_KEY} with the specified 
 * number of directory entries. Also initialize dfs.blockreport.intervalMsec.
 */
public static Configuration initializeStorageStateConf(int numDirs,
                                                       Configuration conf) {
  StringBuffer nameNodeDirs =
    new StringBuffer(new File(TEST_ROOT_DIR, "name1").toString());
  StringBuffer dataNodeDirs =
    new StringBuffer(new File(TEST_ROOT_DIR, "data1").toString());
  for (int i = 2; i <= numDirs; i++) {
    nameNodeDirs.append("," + new File(TEST_ROOT_DIR, "name"+i));
    dataNodeDirs.append("," + new File(TEST_ROOT_DIR, "data"+i));
  }
  if (conf == null) {
    conf = new HdfsConfiguration();
  }
  conf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY, nameNodeDirs.toString());
  conf.set(DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY, nameNodeDirs.toString());
  conf.set(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY, dataNodeDirs.toString());
  conf.setInt(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 10000);
  return conf;
}
 

import org.apache.hadoop.conf.Configuration; //导入依赖的package包/类
public MockContainer(ApplicationAttemptId appAttemptId,
    Dispatcher dispatcher, Configuration conf, String user,
    ApplicationId appId, int uniqId) throws IOException{

  this.user = user;
  this.recordFactory = RecordFactoryProvider.getRecordFactory(conf);
  this.id = BuilderUtils.newContainerId(recordFactory, appId, appAttemptId,
      uniqId);
  this.launchContext = recordFactory
      .newRecordInstance(ContainerLaunchContext.class);
  long currentTime = System.currentTimeMillis();
  this.containerTokenIdentifier =
      BuilderUtils.newContainerTokenIdentifier(BuilderUtils
        .newContainerToken(id, "127.0.0.1", 1234, user,
          BuilderUtils.newResource(1024, 1), currentTime + 10000, 123,
          "password".getBytes(), currentTime));
  this.state = ContainerState.NEW;
}
 

import org.apache.hadoop.conf.Configuration; //导入依赖的package包/类
@Override public int run(String[] args) throws Exception {
  if (args.length < 1) {
    System.err.println("Usage: Clean <output dir>");
    return -1;
  }

  Path p = new Path(args[0]);
  Configuration conf = getConf();
  TableName tableName = getTableName(conf);
  try (FileSystem fs = HFileSystem.get(conf);
      Connection conn = ConnectionFactory.createConnection(conf);
      Admin admin = conn.getAdmin()) {
    if (admin.tableExists(tableName)) {
      admin.disableTable(tableName);
      admin.deleteTable(tableName);
    }

    if (fs.exists(p)) {
      fs.delete(p, true);
    }
  }

  return 0;
}
 

import org.apache.hadoop.conf.Configuration; //导入依赖的package包/类
/**
 * Similar to the above test case, except that it uses a different API to add
 * the filter. Regression test for HADOOP-8786.
 */
@Test
public void testContextSpecificServletFilterWhenInitThrowsException()
    throws Exception {
  Configuration conf = new Configuration();
  HttpServer2 http = createTestServer(conf);
  HttpServer2.defineFilter(http.webAppContext,
      "ErrorFilter", ErrorFilter.class.getName(),
      null, null);
  try {
    http.start();
    fail("expecting exception");
  } catch (IOException e) {
    GenericTestUtils.assertExceptionContains(
        "Unable to initialize WebAppContext", e);
  }
}
 

import org.apache.hadoop.conf.Configuration; //导入依赖的package包/类
public void setConf(Configuration conf) {
  if (conf instanceof JobConf) {
    this.conf = (JobConf) conf;
  } else {
    this.conf = new JobConf(conf);
  }
  this.mapOutputFile = ReflectionUtils.newInstance(
      conf.getClass(MRConfig.TASK_LOCAL_OUTPUT_CLASS,
        MROutputFiles.class, MapOutputFile.class), conf);
  this.lDirAlloc = new LocalDirAllocator(MRConfig.LOCAL_DIR);
  // add the static resolutions (this is required for the junit to
  // work on testcases that simulate multiple nodes on a single physical
  // node.
  String hostToResolved[] = conf.getStrings(MRConfig.STATIC_RESOLUTIONS);
  if (hostToResolved != null) {
    for (String str : hostToResolved) {
      String name = str.substring(0, str.indexOf('='));
      String resolvedName = str.substring(str.indexOf('=') + 1);
      NetUtils.addStaticResolution(name, resolvedName);
    }
  }
}
 

import org.apache.hadoop.conf.Configuration; //导入依赖的package包/类
@BeforeClass
public static void setupBeforeClass() throws Exception {
  TEST_UTIL = new HBaseTestingUtility();
  Configuration conf = TEST_UTIL.getConfiguration();
  // Up the handlers; this test needs more than usual.
  conf.setInt(HConstants.REGION_SERVER_HIGH_PRIORITY_HANDLER_COUNT, 10);
  enableSecurity(conf);
  verifyConfiguration(conf);

  // We expect 0.98 scanning semantics
  conf.setBoolean(AccessControlConstants.CF_ATTRIBUTE_EARLY_OUT, false);

  TEST_UTIL.startMiniCluster();
  TEST_UTIL.waitTableEnabled(AccessControlLists.ACL_TABLE_NAME.getName(), 50000);

  READER = User.createUserForTesting(conf, "reader", new String[0]);
  LIMITED = User.createUserForTesting(conf, "limited", new String[0]);
  DENIED = User.createUserForTesting(conf, "denied", new String[0]);
}
 

import org.apache.hadoop.conf.Configuration; //导入依赖的package包/类
/**
 * Package private, for testability.
 *
 * @param nMaps The number of maps requested for.
 * @param nRecords The number of records to be copied.
 * @param conf The configuration set by users.
 * @return The number of splits each map should handle, ideally.
 */
static int getSplitRatio(int nMaps, int nRecords, Configuration conf) {
  int maxChunksIdeal = getMaxChunksIdeal(conf);
  int minRecordsPerChunk = getMinRecordsPerChunk(conf);
  int splitRatio = getSplitRatio(conf);

  if (nMaps == 1) {
    LOG.warn("nMaps == 1. Why use DynamicInputFormat?");
    return 1;
  }

  if (nMaps > maxChunksIdeal) {
    return splitRatio;
  }

  int nPickups = (int) Math.ceil((float) maxChunksIdeal / nMaps);
  int nRecordsPerChunk = (int) Math.ceil((float) nRecords / (nMaps * nPickups));

  return nRecordsPerChunk < minRecordsPerChunk ? splitRatio : nPickups;
}
 

import org.apache.hadoop.conf.Configuration; //导入依赖的package包/类
@Test
public void testGetPassword() throws Exception {
  Configuration conf = HBaseConfiguration.create();
  conf.set(ReflectiveCredentialProviderClient.CREDENTIAL_PROVIDER_PATH, "jceks://file"
      + new File(UTIL.getDataTestDir().toUri().getPath(), "foo.jks").getCanonicalPath());
  ReflectiveCredentialProviderClient client = new ReflectiveCredentialProviderClient();
  if (client.isHadoopCredentialProviderAvailable()) {
    char[] keyPass = { 'k', 'e', 'y', 'p', 'a', 's', 's' };
    char[] storePass = { 's', 't', 'o', 'r', 'e', 'p', 'a', 's', 's' };
    client.createEntry(conf, "ssl.keypass.alias", keyPass);
    client.createEntry(conf, "ssl.storepass.alias", storePass);

    String keypass = HBaseConfiguration.getPassword(conf, "ssl.keypass.alias", null);
    assertEquals(keypass, new String(keyPass));

    String storepass = HBaseConfiguration.getPassword(conf, "ssl.storepass.alias", null);
    assertEquals(storepass, new String(storePass));
  }
}