Java ScriptBasedMapping类代码示例

import org.apache.hadoop.net.ScriptBasedMapping; //导入依赖的package包/类
public synchronized static void init(Configuration conf) {
  if (initCalled) {
    return;
  } else {
    initCalled = true;
  }
  Class<? extends DNSToSwitchMapping> dnsToSwitchMappingClass =
    conf.getClass(
      CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY, 
      ScriptBasedMapping.class,
      DNSToSwitchMapping.class);
  try {
    DNSToSwitchMapping newInstance = ReflectionUtils.newInstance(
        dnsToSwitchMappingClass, conf);
    // Wrap around the configured class with the Cached implementation so as
    // to save on repetitive lookups.
    // Check if the impl is already caching, to avoid double caching.
    dnsToSwitchMapping =
        ((newInstance instanceof CachedDNSToSwitchMapping) ? newInstance
            : new CachedDNSToSwitchMapping(newInstance));
  } catch (Exception e) {
    throw new RuntimeException(e);
  }
}
 

import org.apache.hadoop.net.ScriptBasedMapping; //导入依赖的package包/类
@Test
public void testScriptName() {
  Configuration conf = new Configuration();
  conf
      .setClass(
          CommonConfigurationKeysPublic.
              NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
          ScriptBasedMapping.class, DNSToSwitchMapping.class);
  conf.set(CommonConfigurationKeysPublic.NET_TOPOLOGY_SCRIPT_FILE_NAME_KEY,
      "testScript");
  RackResolver.init(conf);
  Assert.assertEquals(RackResolver.getDnsToSwitchMapping().toString(),
      "script-based mapping with script testScript");
}
 

import org.apache.hadoop.net.ScriptBasedMapping; //导入依赖的package包/类
/**
 * Activate FSNamesystem daemons.
 */
void activate(Configuration conf) throws IOException {
  setBlockTotal();
  blockManager.activate();
  this.hbthread = new Daemon(new HeartbeatMonitor());
  this.lmthread = new Daemon(leaseManager.new Monitor());
  this.replthread = new Daemon(new ReplicationMonitor());
  hbthread.start();
  lmthread.start();
  replthread.start();

  this.dnthread = new Daemon(new DecommissionManager(this).new Monitor(
      conf.getInt("dfs.namenode.decommission.interval", 30),
      conf.getInt("dfs.namenode.decommission.nodes.per.interval", 5)));
  dnthread.start();

  this.dnsToSwitchMapping = ReflectionUtils.newInstance(
      conf.getClass(DFSConfigKeys.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY, 
                    ScriptBasedMapping.class,
          DNSToSwitchMapping.class), conf);
  
  /* If the dns to swith mapping supports cache, resolve network 
   * locations of those hosts in the include list, 
   * and store the mapping in the cache; so future calls to resolve
   * will be fast.
   */
  if (dnsToSwitchMapping instanceof CachedDNSToSwitchMapping) {
    dnsToSwitchMapping.resolve(new ArrayList<String>(hostsReader.getHosts()));
  }
  registerMXBean();
}
 

import org.apache.hadoop.net.ScriptBasedMapping; //导入依赖的package包/类
public TopologyCache(Configuration conf) {
  super(conf);

  dnsToSwitchMapping = ReflectionUtils.newInstance
    (conf.getClass("topology.node.switch.mapping.impl", ScriptBasedMapping.class,
                   DNSToSwitchMapping.class), conf);
  LOG.info("DnsToSwitchMapping class = " + dnsToSwitchMapping.getClass().getName());
}
 

import org.apache.hadoop.net.ScriptBasedMapping; //导入依赖的package包/类
/**
 * Initialize the filesystem store -this creates the REST client binding.
 *
 * @param fsURI         URI of the filesystem, which is used to map to the filesystem-specific
 *                      options in the configuration file
 * @param configuration configuration
 * @throws IOException on any failure.
 */
public void initialize(URI fsURI, Configuration configuration) throws IOException {
  this.uri = fsURI;
  dnsToSwitchMapping = ReflectionUtils.newInstance(
      configuration.getClass("topology.node.switch.mapping.impl", ScriptBasedMapping.class,
          DNSToSwitchMapping.class), configuration);

  this.swiftRestClient = SwiftRestClient.getInstance(fsURI, configuration);
}
 

import org.apache.hadoop.net.ScriptBasedMapping; //导入依赖的package包/类
public RackManager(Configuration conf) {
  switchMapping = ReflectionUtils.instantiateWithCustomCtor(
      conf.getClass("hbase.util.ip.to.rack.determiner", ScriptBasedMapping.class,
           DNSToSwitchMapping.class).getName(), new Class<?>[]{Configuration.class},
             new Object[]{conf});
}
 

import org.apache.hadoop.net.ScriptBasedMapping; //导入依赖的package包/类
DatanodeManager(final BlockManager blockManager, final Namesystem namesystem,
    final Configuration conf) throws IOException {
  this.namesystem = namesystem;
  this.blockManager = blockManager;
  
  this.heartbeatManager = new HeartbeatManager(namesystem, blockManager, conf);

  networktopology = NetworkTopology.getInstance(conf);

  this.defaultXferPort = NetUtils.createSocketAddr(
        conf.get(DFSConfigKeys.DFS_DATANODE_ADDRESS_KEY,
            DFSConfigKeys.DFS_DATANODE_ADDRESS_DEFAULT)).getPort();
  this.defaultInfoPort = NetUtils.createSocketAddr(
        conf.get(DFSConfigKeys.DFS_DATANODE_HTTP_ADDRESS_KEY,
            DFSConfigKeys.DFS_DATANODE_HTTPS_ADDRESS_DEFAULT)).getPort();
  this.defaultIpcPort = NetUtils.createSocketAddr(
        conf.get(DFSConfigKeys.DFS_DATANODE_IPC_ADDRESS_KEY,
            DFSConfigKeys.DFS_DATANODE_IPC_ADDRESS_DEFAULT)).getPort();
  try {
    this.hostFileManager.refresh(conf.get(DFSConfigKeys.DFS_HOSTS, ""),
      conf.get(DFSConfigKeys.DFS_HOSTS_EXCLUDE, ""));
  } catch (IOException e) {
    LOG.error("error reading hosts files: ", e);
  }

  this.dnsToSwitchMapping = ReflectionUtils.newInstance(
      conf.getClass(DFSConfigKeys.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY, 
          ScriptBasedMapping.class, DNSToSwitchMapping.class), conf);
  
  // If the dns to switch mapping supports cache, resolve network
  // locations of those hosts in the include list and store the mapping
  // in the cache; so future calls to resolve will be fast.
  if (dnsToSwitchMapping instanceof CachedDNSToSwitchMapping) {
    final ArrayList<String> locations = new ArrayList<String>();
    for (Entry entry : hostFileManager.getIncludes()) {
      if (!entry.getIpAddress().isEmpty()) {
        locations.add(entry.getIpAddress());
      }
    }
    dnsToSwitchMapping.resolve(locations);
  };

  final long heartbeatIntervalSeconds = conf.getLong(
      DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY,
      DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_DEFAULT);
  final int heartbeatRecheckInterval = conf.getInt(
      DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY, 
      DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_DEFAULT); // 5 minutes
  this.heartbeatExpireInterval = 2 * heartbeatRecheckInterval
      + 10 * 1000 * heartbeatIntervalSeconds;
  final int blockInvalidateLimit = Math.max(20*(int)(heartbeatIntervalSeconds),
      DFSConfigKeys.DFS_BLOCK_INVALIDATE_LIMIT_DEFAULT);
  this.blockInvalidateLimit = conf.getInt(
      DFSConfigKeys.DFS_BLOCK_INVALIDATE_LIMIT_KEY, blockInvalidateLimit);
  LOG.info(DFSConfigKeys.DFS_BLOCK_INVALIDATE_LIMIT_KEY
      + "=" + this.blockInvalidateLimit);

  this.avoidStaleDataNodesForRead = conf.getBoolean(
      DFSConfigKeys.DFS_NAMENODE_AVOID_STALE_DATANODE_FOR_READ_KEY,
      DFSConfigKeys.DFS_NAMENODE_AVOID_STALE_DATANODE_FOR_READ_DEFAULT);
  this.avoidStaleDataNodesForWrite = conf.getBoolean(
      DFSConfigKeys.DFS_NAMENODE_AVOID_STALE_DATANODE_FOR_WRITE_KEY,
      DFSConfigKeys.DFS_NAMENODE_AVOID_STALE_DATANODE_FOR_WRITE_DEFAULT);
  this.staleInterval = getStaleIntervalFromConf(conf, heartbeatExpireInterval);
  this.ratioUseStaleDataNodesForWrite = conf.getFloat(
      DFSConfigKeys.DFS_NAMENODE_USE_STALE_DATANODE_FOR_WRITE_RATIO_KEY,
      DFSConfigKeys.DFS_NAMENODE_USE_STALE_DATANODE_FOR_WRITE_RATIO_DEFAULT);
  Preconditions.checkArgument(
      (ratioUseStaleDataNodesForWrite > 0 && 
          ratioUseStaleDataNodesForWrite <= 1.0f),
      DFSConfigKeys.DFS_NAMENODE_USE_STALE_DATANODE_FOR_WRITE_RATIO_KEY +
      " = '" + ratioUseStaleDataNodesForWrite + "' is invalid. " +
      "It should be a positive non-zero float value, not greater than 1.0f.");
}
 

import org.apache.hadoop.net.ScriptBasedMapping; //导入依赖的package包/类
DatanodeManager(final BlockManager blockManager, final Namesystem namesystem,
    final Configuration conf) throws IOException {
  this.namesystem = namesystem;
  this.blockManager = blockManager;

  this.networktopology = NetworkTopology.getInstance(conf);

  this.heartbeatManager =
      new HeartbeatManager(namesystem, blockManager, conf);

  this.hostsReader =
      new HostsFileReader(conf.get(DFSConfigKeys.DFS_HOSTS, ""),
          conf.get(DFSConfigKeys.DFS_HOSTS_EXCLUDE, ""));

  this.dnsToSwitchMapping = ReflectionUtils.newInstance(
      conf.getClass(DFSConfigKeys.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
          ScriptBasedMapping.class, DNSToSwitchMapping.class), conf);
  
  // If the dns to switch mapping supports cache, resolve network
  // locations of those hosts in the include list and store the mapping
  // in the cache; so future calls to resolve will be fast.
  if (dnsToSwitchMapping instanceof CachedDNSToSwitchMapping) {
    dnsToSwitchMapping.resolve(new ArrayList<>(hostsReader.getHosts()));
  }
  
  final long heartbeatIntervalSeconds =
      conf.getLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY,
          DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_DEFAULT);
  final int heartbeatRecheckInterval =
      conf.getInt(DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY,
          DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_DEFAULT); // 5 minutes
  this.heartbeatExpireInterval =
      2 * heartbeatRecheckInterval + 10 * 1000 * heartbeatIntervalSeconds;
  final int blockInvalidateLimit =
      Math.max(20 * (int) (heartbeatIntervalSeconds),
          DFSConfigKeys.DFS_BLOCK_INVALIDATE_LIMIT_DEFAULT);
  this.blockInvalidateLimit =
      conf.getInt(DFSConfigKeys.DFS_BLOCK_INVALIDATE_LIMIT_KEY,
          blockInvalidateLimit);
  LOG.info(DFSConfigKeys.DFS_BLOCK_INVALIDATE_LIMIT_KEY + "=" +
      this.blockInvalidateLimit);

  this.avoidStaleDataNodesForRead = conf.getBoolean(
      DFSConfigKeys.DFS_NAMENODE_AVOID_STALE_DATANODE_FOR_READ_KEY,
      DFSConfigKeys.DFS_NAMENODE_AVOID_STALE_DATANODE_FOR_READ_DEFAULT);
  this.avoidStaleDataNodesForWrite = conf.getBoolean(
      DFSConfigKeys.DFS_NAMENODE_AVOID_STALE_DATANODE_FOR_WRITE_KEY,
      DFSConfigKeys.DFS_NAMENODE_AVOID_STALE_DATANODE_FOR_WRITE_DEFAULT);
  this.staleInterval =
      getStaleIntervalFromConf(conf, heartbeatExpireInterval);
  this.ratioUseStaleDataNodesForWrite = conf.getFloat(
      DFSConfigKeys.DFS_NAMENODE_USE_STALE_DATANODE_FOR_WRITE_RATIO_KEY,
      DFSConfigKeys.DFS_NAMENODE_USE_STALE_DATANODE_FOR_WRITE_RATIO_DEFAULT);
  Preconditions.checkArgument((ratioUseStaleDataNodesForWrite > 0 &&
          ratioUseStaleDataNodesForWrite <= 1.0f),
      DFSConfigKeys.DFS_NAMENODE_USE_STALE_DATANODE_FOR_WRITE_RATIO_KEY +
          " = '" + ratioUseStaleDataNodesForWrite + "' is invalid. " +
          "It should be a positive non-zero float value, not greater than 1.0f.");
  
  this.storageIdMap = new StorageIdMap();
}
 

import org.apache.hadoop.net.ScriptBasedMapping; //导入依赖的package包/类
/**
 * Initialize FSNamesystem.
 */
private void initialize(NameNode nn, Configuration conf) throws IOException {
  this.systemStart = now();
  setConfigurationParameters(conf);
  dtSecretManager = createDelegationTokenSecretManager(conf);

  this.nameNodeAddress = nn.getNameNodeAddress();
  this.registerMBean(conf); // register the MBean for the FSNamesystemStutus
  this.dir = new FSDirectory(this, conf);
  StartupOption startOpt = NameNode.getStartupOption(conf);
  this.dir.loadFSImage(getNamespaceDirs(conf),
                       getNamespaceEditsDirs(conf), startOpt);
  long timeTakenToLoadFSImage = now() - systemStart;
  LOG.info("Finished loading FSImage in " + timeTakenToLoadFSImage + " msecs");
  NameNode.getNameNodeMetrics().setFsImageLoadTime(timeTakenToLoadFSImage);
  this.safeMode = new SafeModeInfo(conf);
  setBlockTotal();
  pendingReplications = new PendingReplicationBlocks(
                          conf.getInt("dfs.replication.pending.timeout.sec", 
                                      -1) * 1000L);
  if (isAccessTokenEnabled) {
    accessTokenHandler = new BlockTokenSecretManager(true,
        accessKeyUpdateInterval, accessTokenLifetime);
  }
  this.hbthread = new Daemon(new HeartbeatMonitor());
  this.lmthread = new Daemon(leaseManager.new Monitor());
  this.replmon = new ReplicationMonitor();
  this.replthread = new Daemon(replmon);
  hbthread.start();
  lmthread.start();
  replthread.start();

  this.hostsReader = new HostsFileReader(conf.get("dfs.hosts",""),
                                         conf.get("dfs.hosts.exclude",""));
  this.dnthread = new Daemon(new DecommissionManager(this).new Monitor(
      conf.getInt("dfs.namenode.decommission.interval", 30),
      conf.getInt("dfs.namenode.decommission.nodes.per.interval", 5)));
  dnthread.start();

  this.dnsToSwitchMapping = ReflectionUtils.newInstance(
      conf.getClass("topology.node.switch.mapping.impl", ScriptBasedMapping.class,
          DNSToSwitchMapping.class), conf);
  
  /* If the dns to swith mapping supports cache, resolve network 
   * locations of those hosts in the include list, 
   * and store the mapping in the cache; so future calls to resolve
   * will be fast.
   */
  if (dnsToSwitchMapping instanceof CachedDNSToSwitchMapping) {
    dnsToSwitchMapping.resolve(new ArrayList<String>(hostsReader.getHosts()));
  }
  
  InetSocketAddress socAddr = NameNode.getAddress(conf);
  this.nameNodeHostName = socAddr.getHostName();
  
  registerWith(DefaultMetricsSystem.INSTANCE);
}
 

import org.apache.hadoop.net.ScriptBasedMapping; //导入依赖的package包/类
/**
 * Initialize FSNamesystem.
 */
private void initialize(NameNode nn, Configuration conf) throws IOException {
  this.systemStart = now();
  setConfigurationParameters(conf);

  this.nameNodeAddress = nn.getNameNodeAddress();
  this.registerMBean(conf); // register the MBean for the FSNamesystemStutus
  this.dir = new FSDirectory(this, conf);
  StartupOption startOpt = NameNode.getStartupOption(conf);
  this.dir.loadFSImage(getNamespaceDirs(conf),
                       getNamespaceEditsDirs(conf), startOpt);
  long timeTakenToLoadFSImage = now() - systemStart;
  LOG.info("Finished loading FSImage in " + timeTakenToLoadFSImage + " msecs");
  NameNode.getNameNodeMetrics().fsImageLoadTime.set(
                            (int) timeTakenToLoadFSImage);
  this.safeMode = new SafeModeInfo(conf);
  setBlockTotal();
  pendingReplications = new PendingReplicationBlocks(
                          conf.getInt("dfs.replication.pending.timeout.sec", 
                                      -1) * 1000L);
  this.hbthread = new Daemon(new HeartbeatMonitor());
  this.lmthread = new Daemon(leaseManager.new Monitor());
  this.replthread = new Daemon(new ReplicationMonitor());
  hbthread.start();
  lmthread.start();
  replthread.start();

  this.hostsReader = new HostsFileReader(conf.get("dfs.hosts",""),
                                         conf.get("dfs.hosts.exclude",""));
  this.dnthread = new Daemon(new DecommissionManager(this).new Monitor(
      conf.getInt("dfs.namenode.decommission.interval", 30),
      conf.getInt("dfs.namenode.decommission.nodes.per.interval", 5)));
  dnthread.start();

  this.dnsToSwitchMapping = ReflectionUtils.newInstance(
      conf.getClass("topology.node.switch.mapping.impl", ScriptBasedMapping.class,
          DNSToSwitchMapping.class), conf);
  
  /* If the dns to swith mapping supports cache, resolve network 
   * locations of those hosts in the include list, 
   * and store the mapping in the cache; so future calls to resolve
   * will be fast.
   */
  if (dnsToSwitchMapping instanceof CachedDNSToSwitchMapping) {
    dnsToSwitchMapping.resolve(new ArrayList<String>(hostsReader.getHosts()));
  }
}