Java HdfsConfiguration.set方法代码示例

import org.apache.hadoop.hdfs.HdfsConfiguration; //导入方法依赖的package包/类
@Test
public void TestGetConfIncludeCommand() throws Exception{
	HdfsConfiguration conf = new HdfsConfiguration();
  // Set up the hosts/exclude files.
  localFileSys = FileSystem.getLocal(conf);
  Path workingDir = localFileSys.getWorkingDirectory();
  Path dir = new Path(workingDir, System.getProperty("test.build.data", "target/test/data") + "/Getconf/");
  Path hostsFile = new Path(dir, "hosts");
  Path excludeFile = new Path(dir, "exclude");
  
  // Setup conf
  conf.set(DFSConfigKeys.DFS_HOSTS, hostsFile.toUri().getPath());
  conf.set(DFSConfigKeys.DFS_HOSTS_EXCLUDE, excludeFile.toUri().getPath());
  writeConfigFile(hostsFile, null);
  writeConfigFile(excludeFile, null);    
  String[] args = {"-includeFile"};
  String ret = runTool(conf, args, true);
  assertEquals(hostsFile.toUri().getPath(),ret.trim());
  cleanupFile(localFileSys, excludeFile.getParent());
}
 

import org.apache.hadoop.hdfs.HdfsConfiguration; //导入方法依赖的package包/类
public static HdfsConfiguration initZeroCopyTest() {
  Assume.assumeTrue(NativeIO.isAvailable());
  Assume.assumeTrue(SystemUtils.IS_OS_UNIX);
  HdfsConfiguration conf = new HdfsConfiguration();
  conf.setBoolean(DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_KEY, true);
  conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCK_SIZE);
  conf.setInt(DFSConfigKeys.DFS_CLIENT_MMAP_CACHE_SIZE, 3);
  conf.setLong(DFSConfigKeys.DFS_CLIENT_MMAP_CACHE_TIMEOUT_MS, 100);
  conf.set(DFSConfigKeys.DFS_DOMAIN_SOCKET_PATH_KEY,
      new File(sockDir.getDir(),
        "TestRequestMmapAccess._PORT.sock").getAbsolutePath());
  conf.setBoolean(DFSConfigKeys.
      DFS_CLIENT_READ_SHORTCIRCUIT_SKIP_CHECKSUM_KEY, true);
  conf.setLong(DFS_HEARTBEAT_INTERVAL_KEY, 1);
  conf.setLong(DFS_CACHEREPORT_INTERVAL_MSEC_KEY, 1000);
  conf.setLong(DFS_NAMENODE_PATH_BASED_CACHE_REFRESH_INTERVAL_MS, 1000);
  return conf;
}
 

import org.apache.hadoop.hdfs.HdfsConfiguration; //导入方法依赖的package包/类
@Test
public void TestGetConfExcludeCommand() throws Exception{
	HdfsConfiguration conf = new HdfsConfiguration();
  // Set up the hosts/exclude files.
  localFileSys = FileSystem.getLocal(conf);
  Path workingDir = localFileSys.getWorkingDirectory();
  Path dir = new Path(workingDir, System.getProperty("test.build.data", "target/test/data") + "/Getconf/");
  Path hostsFile = new Path(dir, "hosts");
  Path excludeFile = new Path(dir, "exclude");
  
  // Setup conf
  conf.set(DFSConfigKeys.DFS_HOSTS, hostsFile.toUri().getPath());
  conf.set(DFSConfigKeys.DFS_HOSTS_EXCLUDE, excludeFile.toUri().getPath());
  writeConfigFile(hostsFile, null);
  writeConfigFile(excludeFile, null);    
  String[] args = {"-excludeFile"};
  String ret = runTool(conf, args, true);
  assertEquals(excludeFile.toUri().getPath(),ret.trim());
  cleanupFile(localFileSys, excludeFile.getParent());
}
 

import org.apache.hadoop.hdfs.HdfsConfiguration; //导入方法依赖的package包/类
@Before
public void setUp() throws Exception {
  config = new HdfsConfiguration();
  hdfsDir = new File(MiniDFSCluster.getBaseDirectory());

  if ( hdfsDir.exists() && !FileUtil.fullyDelete(hdfsDir) ) {
    throw new IOException("Could not delete hdfs directory '" + hdfsDir + "'");
  }
  LOG.info("--hdfsdir is " + hdfsDir.getAbsolutePath());
  config.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY,
      fileAsURI(new File(hdfsDir, "name")).toString());
  config.set(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY,
      new File(hdfsDir, "data").getPath());
  config.set(DFSConfigKeys.DFS_DATANODE_ADDRESS_KEY, "0.0.0.0:0");
  config.set(DFSConfigKeys.DFS_DATANODE_HTTP_ADDRESS_KEY, "0.0.0.0:0");
  config.set(DFSConfigKeys.DFS_DATANODE_IPC_ADDRESS_KEY, "0.0.0.0:0");
  config.set(DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_DIR_KEY,
      fileAsURI(new File(hdfsDir, "secondary")).toString());
  config.set(DFSConfigKeys.DFS_NAMENODE_SECONDARY_HTTP_ADDRESS_KEY,
      WILDCARD_HTTP_HOST + "0");
  
  FileSystem.setDefaultUri(config, "hdfs://"+NAME_NODE_HOST + "0");
}
 

import org.apache.hadoop.hdfs.HdfsConfiguration; //导入方法依赖的package包/类
/**
 * Test that the fencing configuration can be overridden per-nameservice
 * or per-namenode
 */
@Test
public void testFencingConfigPerNameNode() throws Exception {
  Mockito.doReturn(STANDBY_READY_RESULT).when(mockProtocol).getServiceStatus();

  final String nsSpecificKey = DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY + "." + NSID;
  final String nnSpecificKey = nsSpecificKey + ".nn1";
  
  HdfsConfiguration conf = getHAConf();
  // Set the default fencer to succeed
  conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, getFencerTrueCommand());
  tool.setConf(conf);
  assertEquals(0, runTool("-failover", "nn1", "nn2", "--forcefence"));
  
  // Set the NN-specific fencer to fail. Should fail to fence.
  conf.set(nnSpecificKey, getFencerFalseCommand());
  tool.setConf(conf);
  assertEquals(-1, runTool("-failover", "nn1", "nn2", "--forcefence"));
  conf.unset(nnSpecificKey);

  // Set an NS-specific fencer to fail. Should fail.
  conf.set(nsSpecificKey, getFencerFalseCommand());
  tool.setConf(conf);
  assertEquals(-1, runTool("-failover", "nn1", "nn2", "--forcefence"));
  
  // Set the NS-specific fencer to succeed. Should succeed
  conf.set(nsSpecificKey, getFencerTrueCommand());
  tool.setConf(conf);
  assertEquals(0, runTool("-failover", "nn1", "nn2", "--forcefence"));
}
 

import org.apache.hadoop.hdfs.HdfsConfiguration; //导入方法依赖的package包/类
@Test
public void testIncludeInternalNameServices() throws Exception {
  final int nsCount = 10;
  final int remoteNsCount = 4;
  HdfsConfiguration conf = new HdfsConfiguration();
  setupNameServices(conf, nsCount);
  setupAddress(conf, DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY, nsCount, 1000);
  setupAddress(conf, DFS_NAMENODE_RPC_ADDRESS_KEY, nsCount, 1500);
  conf.set(DFS_INTERNAL_NAMESERVICES_KEY, "ns1");
  setupStaticHostResolution(nsCount);

  String[] includedNN = new String[] {"nn1:1001"};
  verifyAddresses(conf, TestType.NAMENODE, false, includedNN);
  verifyAddresses(conf, TestType.NNRPCADDRESSES, true, includedNN);
}
 

import org.apache.hadoop.hdfs.HdfsConfiguration; //导入方法依赖的package包/类
@Before
public void setUp() throws Exception {
  config = new HdfsConfiguration();
  config.setBoolean(DFSConfigKeys.DFS_WEBHDFS_ENABLED_KEY, true);
  config.setLong(DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_MAX_LIFETIME_KEY, 10000);
  config.setLong(DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_RENEW_INTERVAL_KEY, 5000);
  config.setBoolean(DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY, true);
  config.set("hadoop.security.auth_to_local",
      "RULE:[2:[email protected]$0]([email protected]*FOO.COM)s/@.*//" + "DEFAULT");
  FileSystem.setDefaultUri(config, "hdfs://localhost:" + "0");
  cluster = new MiniDFSCluster.Builder(config).numDataNodes(0).build();
  cluster.waitActive();
  dtSecretManager = NameNodeAdapter.getDtSecretManager(
      cluster.getNamesystem());
}
 

import org.apache.hadoop.hdfs.HdfsConfiguration; //导入方法依赖的package包/类
@Test
public void testFailoverWithFencerConfiguredAndForce() throws Exception {
  Mockito.doReturn(STANDBY_READY_RESULT).when(mockProtocol).getServiceStatus();
  HdfsConfiguration conf = getHAConf();
  conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, getFencerTrueCommand());
  tool.setConf(conf);
  assertEquals(0, runTool("-failover", "nn1", "nn2", "--forcefence"));
}
 

import org.apache.hadoop.hdfs.HdfsConfiguration; //导入方法依赖的package包/类
private HdfsConfiguration getHAConf() {
  HdfsConfiguration conf = new HdfsConfiguration();
  conf.set(DFSConfigKeys.DFS_NAMESERVICES, NSID);    
  conf.set(DFSConfigKeys.DFS_NAMESERVICE_ID, NSID);
  conf.set(DFSUtil.addKeySuffixes(
      DFSConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX, NSID), "nn1,nn2");    
  conf.set(DFSConfigKeys.DFS_HA_NAMENODE_ID_KEY, "nn1");
  conf.set(DFSUtil.addKeySuffixes(
          DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY, NSID, "nn1"),
      HOST_A + ":12345");
  conf.set(DFSUtil.addKeySuffixes(
          DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY, NSID, "nn2"),
      HOST_B + ":12345");
  return conf;
}
 

import org.apache.hadoop.hdfs.HdfsConfiguration; //导入方法依赖的package包/类
@Test
public void testForceFenceOptionListedBeforeArgs() throws Exception {
  Mockito.doReturn(STANDBY_READY_RESULT).when(mockProtocol).getServiceStatus();
  HdfsConfiguration conf = getHAConf();
  conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, getFencerTrueCommand());
  tool.setConf(conf);
  assertEquals(0, runTool("-failover", "--forcefence", "nn1", "nn2"));
}
 

import org.apache.hadoop.hdfs.HdfsConfiguration; //导入方法依赖的package包/类
@Test
public void testPrivacy() throws Exception {
  HdfsConfiguration clusterConf = createSecureConfig(
    "authentication,integrity,privacy");
  startCluster(clusterConf);
  HdfsConfiguration clientConf = new HdfsConfiguration(clusterConf);
  clientConf.set(DFS_DATA_TRANSFER_PROTECTION_KEY, "privacy");
  doTest(clientConf);
}
 

import org.apache.hadoop.hdfs.HdfsConfiguration; //导入方法依赖的package包/类
@Test
public void testClientAndServerDoNotHaveCommonQop() throws Exception {
  HdfsConfiguration clusterConf = createSecureConfig("privacy");
  startCluster(clusterConf);
  HdfsConfiguration clientConf = new HdfsConfiguration(clusterConf);
  clientConf.set(DFS_DATA_TRANSFER_PROTECTION_KEY, "authentication");
  exception.expect(IOException.class);
  exception.expectMessage("could only be replicated to 0 nodes");
  doTest(clientConf);
}
 

import org.apache.hadoop.hdfs.HdfsConfiguration; //导入方法依赖的package包/类
@Test
public void testServerSaslNoClientSasl() throws Exception {
  HdfsConfiguration clusterConf = createSecureConfig(
    "authentication,integrity,privacy");
  // Set short retry timeouts so this test runs faster
  clusterConf.setInt(DFSConfigKeys.DFS_CLIENT_RETRY_WINDOW_BASE, 10);
  startCluster(clusterConf);
  HdfsConfiguration clientConf = new HdfsConfiguration(clusterConf);
  clientConf.set(DFS_DATA_TRANSFER_PROTECTION_KEY, "");

  LogCapturer logs = GenericTestUtils.LogCapturer.captureLogs(
      LogFactory.getLog(DataNode.class));
  try {
    doTest(clientConf);
    Assert.fail("Should fail if SASL data transfer protection is not " +
        "configured or not supported in client");
  } catch (IOException e) {
    GenericTestUtils.assertMatches(e.getMessage(), 
        "could only be replicated to 0 nodes");
  } finally {
    logs.stopCapturing();
  }

  GenericTestUtils.assertMatches(logs.getOutput(),
      "Failed to read expected SASL data transfer protection " +
      "handshake from client at");
}
 

import org.apache.hadoop.hdfs.HdfsConfiguration; //导入方法依赖的package包/类
@Test
public void testDataNodeAbortsIfNotHttpsOnly() throws Exception {
  HdfsConfiguration clusterConf = createSecureConfig("authentication");
  clusterConf.set(DFS_HTTP_POLICY_KEY,
    HttpConfig.Policy.HTTP_AND_HTTPS.name());
  exception.expect(RuntimeException.class);
  exception.expectMessage("Cannot start secure DataNode");
  startCluster(clusterConf);
}
 

import org.apache.hadoop.hdfs.HdfsConfiguration; //导入方法依赖的package包/类
@Test
public void testFailoverWithFencerConfigured() throws Exception {
  Mockito.doReturn(STANDBY_READY_RESULT).when(mockProtocol).getServiceStatus();
  HdfsConfiguration conf = getHAConf();
  conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, getFencerTrueCommand());
  tool.setConf(conf);
  assertEquals(0, runTool("-failover", "nn1", "nn2"));
}