Java CachedDNSToSwitchMapping类代码示例

import org.apache.hadoop.net.CachedDNSToSwitchMapping; //导入依赖的package包/类
public synchronized static void init(Configuration conf) {
  if (initCalled) {
    return;
  } else {
    initCalled = true;
  }
  Class<? extends DNSToSwitchMapping> dnsToSwitchMappingClass =
    conf.getClass(
      CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY, 
      ScriptBasedMapping.class,
      DNSToSwitchMapping.class);
  try {
    DNSToSwitchMapping newInstance = ReflectionUtils.newInstance(
        dnsToSwitchMappingClass, conf);
    // Wrap around the configured class with the Cached implementation so as
    // to save on repetitive lookups.
    // Check if the impl is already caching, to avoid double caching.
    dnsToSwitchMapping =
        ((newInstance instanceof CachedDNSToSwitchMapping) ? newInstance
            : new CachedDNSToSwitchMapping(newInstance));
  } catch (Exception e) {
    throw new RuntimeException(e);
  }
}
 

import org.apache.hadoop.net.CachedDNSToSwitchMapping; //导入依赖的package包/类
private String resolveNetworkLocation (DatanodeID node) {
  List<String> names = new ArrayList<String>(1);
  if (dnsToSwitchMapping instanceof CachedDNSToSwitchMapping) {
    names.add(node.getIpAddr());
  } else {
    names.add(node.getHostName());
  }
  
  // resolve its network location
  List<String> rName = dnsToSwitchMapping.resolve(names);
  String networkLocation;
  if (rName == null) {
    LOG.error("The resolve call returned null! Using " + 
        NetworkTopology.DEFAULT_RACK + " for host " + names);
    networkLocation = NetworkTopology.DEFAULT_RACK;
  } else {
    networkLocation = rName.get(0);
  }
  return networkLocation;
}
 

import org.apache.hadoop.net.CachedDNSToSwitchMapping; //导入依赖的package包/类
private void resolveNetworkLocation(DatanodeDescriptor node) {
  List<String> names = new ArrayList<>(1);
  if (dnsToSwitchMapping instanceof CachedDNSToSwitchMapping) {
    names.add(node.getIpAddr());
  } else {
    names.add(node.getHostName());
  }
  
  // resolve its network location
  List<String> rName = dnsToSwitchMapping.resolve(names);
  String networkLocation;
  if (rName == null) {
    LOG.error("The resolve call returned null! Using " +
        NetworkTopology.DEFAULT_RACK + " for host " + names);
    networkLocation = NetworkTopology.DEFAULT_RACK;
  } else {
    networkLocation = rName.get(0);
  }
  node.setNetworkLocation(networkLocation);
}
 

import org.apache.hadoop.net.CachedDNSToSwitchMapping; //导入依赖的package包/类
private void resolveNetworkLocation (DatanodeDescriptor node) {
  List<String> names = new ArrayList<String>(1);
  if (dnsToSwitchMapping instanceof CachedDNSToSwitchMapping) {
    // get the node's IP address
    names.add(node.getHost());
  } else {
    // get the node's host name
    String hostName = node.getHostName();
    int colon = hostName.indexOf(":");
    hostName = (colon==-1)?hostName:hostName.substring(0,colon);
    names.add(hostName);
  }
  
  // resolve its network location
  List<String> rName = dnsToSwitchMapping.resolve(names);
  String networkLocation;
  if (rName == null) {
    LOG.error("The resolve call returned null! Using " + 
        NetworkTopology.DEFAULT_RACK + " for host " + names);
    networkLocation = NetworkTopology.DEFAULT_RACK;
  } else {
    networkLocation = rName.get(0);
  }
  node.setNetworkLocation(networkLocation);
}
 

import org.apache.hadoop.net.CachedDNSToSwitchMapping; //导入依赖的package包/类
void resolveNetworkLocation(DatanodeInfo node) {
  List<String> names = new ArrayList<String>(1);
  if (dnsToSwitchMapping instanceof CachedDNSToSwitchMapping) {
    // get the node's IP address
    names.add(node.getHost());
  } else {
    // get the node's host name
    String hostName = node.getHostName();
    int colon = hostName.indexOf(":");
    hostName = (colon == -1) ? hostName : hostName.substring(0, colon);
    names.add(hostName);
  }

  // resolve its network location
  List<String> rName = dnsToSwitchMapping.resolve(names);
  String networkLocation;
  if (rName == null) {
    LOG.error("The resolve call returned null! Using " +
      NetworkTopology.DEFAULT_RACK + " for host " + names);
    networkLocation = NetworkTopology.DEFAULT_RACK;
  } else {
    networkLocation = rName.get(0);
  }
  node.setNetworkLocation(networkLocation);
}
 

import org.apache.hadoop.net.CachedDNSToSwitchMapping; //导入依赖的package包/类
void resolveNetworkLocation(DatanodeInfo node) {
  List<String> names = new ArrayList<String>(1);
  if (dnsToSwitchMapping instanceof CachedDNSToSwitchMapping) {
    // get the node's IP address
    names.add(node.getHost());
  } else if (dnsToSwitchMapping instanceof StaticMapping) {
    names.add(toHostPort(node));
  } else {
    // get the node's host name
    String hostName = node.getHostName();
    int colon = hostName.indexOf(":");
    hostName = (colon == -1) ? hostName : hostName.substring(0, colon);
    names.add(hostName);
  }

  // resolve its network location
  List<String> rName = dnsToSwitchMapping.resolve(names);
  String networkLocation;
  if (rName == null) {
    LOG.error("The resolve call returned null! Using " +
      NetworkTopology.DEFAULT_RACK + " for host " + names);
    networkLocation = NetworkTopology.DEFAULT_RACK;
  } else {
    networkLocation = rName.get(0);
  }
  node.setNetworkLocation(networkLocation);
}
 

import org.apache.hadoop.net.CachedDNSToSwitchMapping; //导入依赖的package包/类
/**
 * Activate FSNamesystem daemons.
 */
void activate(Configuration conf) throws IOException {
  setBlockTotal();
  blockManager.activate();
  this.hbthread = new Daemon(new HeartbeatMonitor());
  this.lmthread = new Daemon(leaseManager.new Monitor());
  this.replthread = new Daemon(new ReplicationMonitor());
  hbthread.start();
  lmthread.start();
  replthread.start();

  this.dnthread = new Daemon(new DecommissionManager(this).new Monitor(
      conf.getInt("dfs.namenode.decommission.interval", 30),
      conf.getInt("dfs.namenode.decommission.nodes.per.interval", 5)));
  dnthread.start();

  this.dnsToSwitchMapping = ReflectionUtils.newInstance(
      conf.getClass(DFSConfigKeys.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY, 
                    ScriptBasedMapping.class,
          DNSToSwitchMapping.class), conf);
  
  /* If the dns to swith mapping supports cache, resolve network 
   * locations of those hosts in the include list, 
   * and store the mapping in the cache; so future calls to resolve
   * will be fast.
   */
  if (dnsToSwitchMapping instanceof CachedDNSToSwitchMapping) {
    dnsToSwitchMapping.resolve(new ArrayList<String>(hostsReader.getHosts()));
  }
  registerMXBean();
}
 

import org.apache.hadoop.net.CachedDNSToSwitchMapping; //导入依赖的package包/类
DatanodeManager(final BlockManager blockManager, final Namesystem namesystem,
    final Configuration conf) throws IOException {
  this.namesystem = namesystem;
  this.blockManager = blockManager;
  
  this.heartbeatManager = new HeartbeatManager(namesystem, blockManager, conf);

  networktopology = NetworkTopology.getInstance(conf);

  this.defaultXferPort = NetUtils.createSocketAddr(
        conf.get(DFSConfigKeys.DFS_DATANODE_ADDRESS_KEY,
            DFSConfigKeys.DFS_DATANODE_ADDRESS_DEFAULT)).getPort();
  this.defaultInfoPort = NetUtils.createSocketAddr(
        conf.get(DFSConfigKeys.DFS_DATANODE_HTTP_ADDRESS_KEY,
            DFSConfigKeys.DFS_DATANODE_HTTPS_ADDRESS_DEFAULT)).getPort();
  this.defaultIpcPort = NetUtils.createSocketAddr(
        conf.get(DFSConfigKeys.DFS_DATANODE_IPC_ADDRESS_KEY,
            DFSConfigKeys.DFS_DATANODE_IPC_ADDRESS_DEFAULT)).getPort();
  try {
    this.hostFileManager.refresh(conf.get(DFSConfigKeys.DFS_HOSTS, ""),
      conf.get(DFSConfigKeys.DFS_HOSTS_EXCLUDE, ""));
  } catch (IOException e) {
    LOG.error("error reading hosts files: ", e);
  }

  this.dnsToSwitchMapping = ReflectionUtils.newInstance(
      conf.getClass(DFSConfigKeys.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY, 
          ScriptBasedMapping.class, DNSToSwitchMapping.class), conf);
  
  // If the dns to switch mapping supports cache, resolve network
  // locations of those hosts in the include list and store the mapping
  // in the cache; so future calls to resolve will be fast.
  if (dnsToSwitchMapping instanceof CachedDNSToSwitchMapping) {
    final ArrayList<String> locations = new ArrayList<String>();
    for (Entry entry : hostFileManager.getIncludes()) {
      if (!entry.getIpAddress().isEmpty()) {
        locations.add(entry.getIpAddress());
      }
    }
    dnsToSwitchMapping.resolve(locations);
  };

  final long heartbeatIntervalSeconds = conf.getLong(
      DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY,
      DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_DEFAULT);
  final int heartbeatRecheckInterval = conf.getInt(
      DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY, 
      DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_DEFAULT); // 5 minutes
  this.heartbeatExpireInterval = 2 * heartbeatRecheckInterval
      + 10 * 1000 * heartbeatIntervalSeconds;
  final int blockInvalidateLimit = Math.max(20*(int)(heartbeatIntervalSeconds),
      DFSConfigKeys.DFS_BLOCK_INVALIDATE_LIMIT_DEFAULT);
  this.blockInvalidateLimit = conf.getInt(
      DFSConfigKeys.DFS_BLOCK_INVALIDATE_LIMIT_KEY, blockInvalidateLimit);
  LOG.info(DFSConfigKeys.DFS_BLOCK_INVALIDATE_LIMIT_KEY
      + "=" + this.blockInvalidateLimit);

  this.avoidStaleDataNodesForRead = conf.getBoolean(
      DFSConfigKeys.DFS_NAMENODE_AVOID_STALE_DATANODE_FOR_READ_KEY,
      DFSConfigKeys.DFS_NAMENODE_AVOID_STALE_DATANODE_FOR_READ_DEFAULT);
  this.avoidStaleDataNodesForWrite = conf.getBoolean(
      DFSConfigKeys.DFS_NAMENODE_AVOID_STALE_DATANODE_FOR_WRITE_KEY,
      DFSConfigKeys.DFS_NAMENODE_AVOID_STALE_DATANODE_FOR_WRITE_DEFAULT);
  this.staleInterval = getStaleIntervalFromConf(conf, heartbeatExpireInterval);
  this.ratioUseStaleDataNodesForWrite = conf.getFloat(
      DFSConfigKeys.DFS_NAMENODE_USE_STALE_DATANODE_FOR_WRITE_RATIO_KEY,
      DFSConfigKeys.DFS_NAMENODE_USE_STALE_DATANODE_FOR_WRITE_RATIO_DEFAULT);
  Preconditions.checkArgument(
      (ratioUseStaleDataNodesForWrite > 0 && 
          ratioUseStaleDataNodesForWrite <= 1.0f),
      DFSConfigKeys.DFS_NAMENODE_USE_STALE_DATANODE_FOR_WRITE_RATIO_KEY +
      " = '" + ratioUseStaleDataNodesForWrite + "' is invalid. " +
      "It should be a positive non-zero float value, not greater than 1.0f.");
}
 

import org.apache.hadoop.net.CachedDNSToSwitchMapping; //导入依赖的package包/类
DatanodeManager(final BlockManager blockManager, final Namesystem namesystem,
    final Configuration conf) throws IOException {
  this.namesystem = namesystem;
  this.blockManager = blockManager;

  this.networktopology = NetworkTopology.getInstance(conf);

  this.heartbeatManager =
      new HeartbeatManager(namesystem, blockManager, conf);

  this.hostsReader =
      new HostsFileReader(conf.get(DFSConfigKeys.DFS_HOSTS, ""),
          conf.get(DFSConfigKeys.DFS_HOSTS_EXCLUDE, ""));

  this.dnsToSwitchMapping = ReflectionUtils.newInstance(
      conf.getClass(DFSConfigKeys.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
          ScriptBasedMapping.class, DNSToSwitchMapping.class), conf);
  
  // If the dns to switch mapping supports cache, resolve network
  // locations of those hosts in the include list and store the mapping
  // in the cache; so future calls to resolve will be fast.
  if (dnsToSwitchMapping instanceof CachedDNSToSwitchMapping) {
    dnsToSwitchMapping.resolve(new ArrayList<>(hostsReader.getHosts()));
  }
  
  final long heartbeatIntervalSeconds =
      conf.getLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY,
          DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_DEFAULT);
  final int heartbeatRecheckInterval =
      conf.getInt(DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY,
          DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_DEFAULT); // 5 minutes
  this.heartbeatExpireInterval =
      2 * heartbeatRecheckInterval + 10 * 1000 * heartbeatIntervalSeconds;
  final int blockInvalidateLimit =
      Math.max(20 * (int) (heartbeatIntervalSeconds),
          DFSConfigKeys.DFS_BLOCK_INVALIDATE_LIMIT_DEFAULT);
  this.blockInvalidateLimit =
      conf.getInt(DFSConfigKeys.DFS_BLOCK_INVALIDATE_LIMIT_KEY,
          blockInvalidateLimit);
  LOG.info(DFSConfigKeys.DFS_BLOCK_INVALIDATE_LIMIT_KEY + "=" +
      this.blockInvalidateLimit);

  this.avoidStaleDataNodesForRead = conf.getBoolean(
      DFSConfigKeys.DFS_NAMENODE_AVOID_STALE_DATANODE_FOR_READ_KEY,
      DFSConfigKeys.DFS_NAMENODE_AVOID_STALE_DATANODE_FOR_READ_DEFAULT);
  this.avoidStaleDataNodesForWrite = conf.getBoolean(
      DFSConfigKeys.DFS_NAMENODE_AVOID_STALE_DATANODE_FOR_WRITE_KEY,
      DFSConfigKeys.DFS_NAMENODE_AVOID_STALE_DATANODE_FOR_WRITE_DEFAULT);
  this.staleInterval =
      getStaleIntervalFromConf(conf, heartbeatExpireInterval);
  this.ratioUseStaleDataNodesForWrite = conf.getFloat(
      DFSConfigKeys.DFS_NAMENODE_USE_STALE_DATANODE_FOR_WRITE_RATIO_KEY,
      DFSConfigKeys.DFS_NAMENODE_USE_STALE_DATANODE_FOR_WRITE_RATIO_DEFAULT);
  Preconditions.checkArgument((ratioUseStaleDataNodesForWrite > 0 &&
          ratioUseStaleDataNodesForWrite <= 1.0f),
      DFSConfigKeys.DFS_NAMENODE_USE_STALE_DATANODE_FOR_WRITE_RATIO_KEY +
          " = '" + ratioUseStaleDataNodesForWrite + "' is invalid. " +
          "It should be a positive non-zero float value, not greater than 1.0f.");
  
  this.storageIdMap = new StorageIdMap();
}
 

import org.apache.hadoop.net.CachedDNSToSwitchMapping; //导入依赖的package包/类
/**
 * Initialize FSNamesystem.
 */
private void initialize(NameNode nn, Configuration conf) throws IOException {
  this.systemStart = now();
  setConfigurationParameters(conf);
  dtSecretManager = createDelegationTokenSecretManager(conf);

  this.nameNodeAddress = nn.getNameNodeAddress();
  this.registerMBean(conf); // register the MBean for the FSNamesystemStutus
  this.dir = new FSDirectory(this, conf);
  StartupOption startOpt = NameNode.getStartupOption(conf);
  this.dir.loadFSImage(getNamespaceDirs(conf),
                       getNamespaceEditsDirs(conf), startOpt);
  long timeTakenToLoadFSImage = now() - systemStart;
  LOG.info("Finished loading FSImage in " + timeTakenToLoadFSImage + " msecs");
  NameNode.getNameNodeMetrics().setFsImageLoadTime(timeTakenToLoadFSImage);
  this.safeMode = new SafeModeInfo(conf);
  setBlockTotal();
  pendingReplications = new PendingReplicationBlocks(
                          conf.getInt("dfs.replication.pending.timeout.sec", 
                                      -1) * 1000L);
  if (isAccessTokenEnabled) {
    accessTokenHandler = new BlockTokenSecretManager(true,
        accessKeyUpdateInterval, accessTokenLifetime);
  }
  this.hbthread = new Daemon(new HeartbeatMonitor());
  this.lmthread = new Daemon(leaseManager.new Monitor());
  this.replmon = new ReplicationMonitor();
  this.replthread = new Daemon(replmon);
  hbthread.start();
  lmthread.start();
  replthread.start();

  this.hostsReader = new HostsFileReader(conf.get("dfs.hosts",""),
                                         conf.get("dfs.hosts.exclude",""));
  this.dnthread = new Daemon(new DecommissionManager(this).new Monitor(
      conf.getInt("dfs.namenode.decommission.interval", 30),
      conf.getInt("dfs.namenode.decommission.nodes.per.interval", 5)));
  dnthread.start();

  this.dnsToSwitchMapping = ReflectionUtils.newInstance(
      conf.getClass("topology.node.switch.mapping.impl", ScriptBasedMapping.class,
          DNSToSwitchMapping.class), conf);
  
  /* If the dns to swith mapping supports cache, resolve network 
   * locations of those hosts in the include list, 
   * and store the mapping in the cache; so future calls to resolve
   * will be fast.
   */
  if (dnsToSwitchMapping instanceof CachedDNSToSwitchMapping) {
    dnsToSwitchMapping.resolve(new ArrayList<String>(hostsReader.getHosts()));
  }
  
  InetSocketAddress socAddr = NameNode.getAddress(conf);
  this.nameNodeHostName = socAddr.getHostName();
  
  registerWith(DefaultMetricsSystem.INSTANCE);
}
 

import org.apache.hadoop.net.CachedDNSToSwitchMapping; //导入依赖的package包/类
/**
 * Initialize FSNamesystem.
 */
private void initialize(NameNode nn, Configuration conf) throws IOException {
  this.systemStart = now();
  setConfigurationParameters(conf);

  this.nameNodeAddress = nn.getNameNodeAddress();
  this.registerMBean(conf); // register the MBean for the FSNamesystemStutus
  this.dir = new FSDirectory(this, conf);
  StartupOption startOpt = NameNode.getStartupOption(conf);
  this.dir.loadFSImage(getNamespaceDirs(conf),
                       getNamespaceEditsDirs(conf), startOpt);
  long timeTakenToLoadFSImage = now() - systemStart;
  LOG.info("Finished loading FSImage in " + timeTakenToLoadFSImage + " msecs");
  NameNode.getNameNodeMetrics().fsImageLoadTime.set(
                            (int) timeTakenToLoadFSImage);
  this.safeMode = new SafeModeInfo(conf);
  setBlockTotal();
  pendingReplications = new PendingReplicationBlocks(
                          conf.getInt("dfs.replication.pending.timeout.sec", 
                                      -1) * 1000L);
  this.hbthread = new Daemon(new HeartbeatMonitor());
  this.lmthread = new Daemon(leaseManager.new Monitor());
  this.replthread = new Daemon(new ReplicationMonitor());
  hbthread.start();
  lmthread.start();
  replthread.start();

  this.hostsReader = new HostsFileReader(conf.get("dfs.hosts",""),
                                         conf.get("dfs.hosts.exclude",""));
  this.dnthread = new Daemon(new DecommissionManager(this).new Monitor(
      conf.getInt("dfs.namenode.decommission.interval", 30),
      conf.getInt("dfs.namenode.decommission.nodes.per.interval", 5)));
  dnthread.start();

  this.dnsToSwitchMapping = ReflectionUtils.newInstance(
      conf.getClass("topology.node.switch.mapping.impl", ScriptBasedMapping.class,
          DNSToSwitchMapping.class), conf);
  
  /* If the dns to swith mapping supports cache, resolve network 
   * locations of those hosts in the include list, 
   * and store the mapping in the cache; so future calls to resolve
   * will be fast.
   */
  if (dnsToSwitchMapping instanceof CachedDNSToSwitchMapping) {
    dnsToSwitchMapping.resolve(new ArrayList<String>(hostsReader.getHosts()));
  }
}