Java SupervisorSummary类代码示例

import backtype.storm.generated.SupervisorSummary; //导入依赖的package包/类
/**
 * Convert thrift SupervisorSummary to UI bean SupervisorSumm
 * 
 * @param ss
 * @return
 */
public static List<SupervisorSumm> supervisorSummary(
		List<SupervisorSummary> ss) {
	// uptime host slots usedslots

	List<SupervisorSumm> ssumm = new ArrayList<SupervisorSumm>();

	if (ss == null) {
		ss = new ArrayList<SupervisorSummary>();
	}

	for (SupervisorSummary s : ss) {
		SupervisorSumm ssum = new SupervisorSumm(s);

		ssumm.add(ssum);
	}

	return ssumm;
}
 

import backtype.storm.generated.SupervisorSummary; //导入依赖的package包/类
/**
 * Returns the mapping of supervisor host names to supervisor ids.
 * 
 * @param summary the cluster summary
 * @return the mapping (a host may run multiple supervisors and then we return them in the sequence given 
 *   by thrift)
 */
public static Map<String, List<String>> getSupervisorHostIdMapping(ClusterSummary summary) {
    Map<String, List<String>> result = new HashMap<String, List<String>>();
    List<SupervisorSummary> supervisors = summary.get_supervisors();
    for (int s = 0; s < summary.get_supervisors_size(); s++) {
        SupervisorSummary supervisor = supervisors.get(s);
        String host = supervisor.get_host();
        List<String> tmp = result.get(host);
        if (null == tmp) {
            tmp = new ArrayList<String>();
            result.put(host, tmp);
        }
        tmp.add(supervisor.get_supervisor_id());
    }
    return result;
}
 

import backtype.storm.generated.SupervisorSummary; //导入依赖的package包/类
public static SupervisorSummary mkSupervisorSummary(
		SupervisorInfo supervisorInfo, String supervisorId,
		Map<String, Integer> supervisorToUsedSlotNum) {
	Integer usedNum = supervisorToUsedSlotNum.get(supervisorId);

	SupervisorSummary summary = new SupervisorSummary(
			supervisorInfo.getHostName(), supervisorId,
			supervisorInfo.getUptimeSecs(), supervisorInfo.getWorkerPorts()
					.size(), usedNum == null ? 0 : usedNum);

	return summary;
}
 

import backtype.storm.generated.SupervisorSummary; //导入依赖的package包/类
public SupervisorSumm(SupervisorSummary s) {
	this.host = NetWorkUtils.ip2Host(s.get_host());
	this.ip = NetWorkUtils.host2Ip(s.get_host());
	this.uptime = StatBuckets.prettyUptimeStr(s.get_uptime_secs());

	this.totalPort = String.valueOf(s.get_num_workers());
	this.usedPort = String.valueOf(s.get_num_used_workers());
}
 

import backtype.storm.generated.SupervisorSummary; //导入依赖的package包/类
public static SupervisorSummary mkSupervisorSummary(SupervisorInfo supervisorInfo, String supervisorId,
                                                    Map<String, Integer> supervisorToUsedSlotNum) {
    Integer usedNum = supervisorToUsedSlotNum.get(supervisorId);

    SupervisorSummary summary = new SupervisorSummary(supervisorInfo.getHostName(), supervisorId,
            supervisorInfo.getUptimeSecs(), supervisorInfo.getWorkerPorts().size(), usedNum == null ? 0 : usedNum);

    return summary;
}
 

import backtype.storm.generated.SupervisorSummary; //导入依赖的package包/类
private void getClusterInfo(Client client) {
    try {
        ClusterSummary clusterSummary = client.getClusterInfo();
        List<SupervisorSummary> supervisorSummaryList = clusterSummary.get_supervisors();
        int totalWorkers = 0;
        int usedWorkers = 0;
        for(SupervisorSummary summary : supervisorSummaryList){
            totalWorkers += summary.get_num_workers() ;
            usedWorkers += summary.get_num_used_workers();
        }
        int freeWorkers = totalWorkers - usedWorkers;
        LOGGER.info("cluster totalWorkers = " + totalWorkers 
                + ", usedWorkers = " + usedWorkers 
                + ", freeWorkers  = " +  freeWorkers);
        
        HttpCatClient.sendMetric("ClusterMonitor", "freeSlots", "avg", String.valueOf(freeWorkers));
        HttpCatClient.sendMetric("ClusterMonitor", "totalSlots", "avg", String.valueOf(totalWorkers));
        
        List<TopologySummary> topologySummaryList = clusterSummary.get_topologies();
        long clusterTPS = 0l;
        for(TopologySummary topology : topologySummaryList){
            long topologyTPS = getTopologyTPS(topology, client);
            clusterTPS += topologyTPS;
            if(topology.get_name().startsWith("ClusterMonitor")){
                continue;
            }
            HttpCatClient.sendMetric(topology.get_name(), topology.get_name() + "-TPS", "avg", String.valueOf(topologyTPS));
        }
        HttpCatClient.sendMetric("ClusterMonitor", "ClusterEmitTPS", "avg", String.valueOf(clusterTPS));
        
    } catch (TException e) {
        initClient(configMap);
        LOGGER.error("get client info error.", e);
    }
    catch(NotAliveException nae){
        LOGGER.warn("topology is dead.", nae);
    }
}
 

import backtype.storm.generated.SupervisorSummary; //导入依赖的package包/类
/**
 * Tests the cluster aggregation.
 */
@Test
public void testAggregation() {
    SystemState state = new SystemState();
    ArrayList<SupervisorSummary> supervisors = new ArrayList<SupervisorSummary>();
    PlatformSystemPart part = state.getPlatform();

    // empty is unrealistic except for killing supervisors or having an empty cluster
    StormClusterMonitoringTask.aggregate(part, supervisors);
    assertEquals(0, part, ResourceUsage.USED_MACHINES);
    
    supervisors.clear();
    supervisors.add(new SupervisorSummary("host1", 10, 1, 1, "host1"));
    supervisors.add(new SupervisorSummary("host2", 10, 1, 1, "host2"));

    StormClusterMonitoringTask.aggregate(part, supervisors);
    assertEquals(2, part, ResourceUsage.USED_MACHINES);

    supervisors.clear();
    supervisors.add(new SupervisorSummary("host1", 10, 1, 1, "host1"));
    supervisors.add(new SupervisorSummary("host2", 10, 1, 0, "host2"));
    
    StormClusterMonitoringTask.aggregate(part, supervisors);
    assertEquals(1, part, ResourceUsage.USED_MACHINES);

    supervisors.clear();
    supervisors.add(new SupervisorSummary("host1", 10, 1, 0, "host1"));
    supervisors.add(new SupervisorSummary("host2", 10, 1, 0, "host2"));
    
    StormClusterMonitoringTask.aggregate(part, supervisors);
    assertEquals(0, part, ResourceUsage.USED_MACHINES);

    
    // empty is unrealistic except for killing supervisors or having an empty cluster
    supervisors.clear();
    StormClusterMonitoringTask.aggregate(part, supervisors);
    assertEquals(0, part, ResourceUsage.USED_MACHINES);

}
 

import backtype.storm.generated.SupervisorSummary; //导入依赖的package包/类
/**
 * Returns the mapping of supervisor ids to host names.
 * 
 * @param summary the cluster summary
 * @return the mapping
 */
public static Map<String, String> getSupervisorIdHostMapping(ClusterSummary summary) {
    Map<String, String> result = new HashMap<String, String>();
    List<SupervisorSummary> supervisors = summary.get_supervisors();
    for (int s = 0; s < summary.get_supervisors_size(); s++) {
        SupervisorSummary supervisor = supervisors.get(s);
        result.put(supervisor.get_supervisor_id(), supervisor.get_host());
    }
    return result;
}
 

import backtype.storm.generated.SupervisorSummary; //导入依赖的package包/类
public static List<String> getSupervisorHosts() throws Exception {
    try {
        List<String> hosts = new ArrayList<>();
        NimbusClient client = getNimbusClient(null);
        ClusterSummary clusterSummary = client.getClient().getClusterInfo();
        List<SupervisorSummary> supervisorSummaries = clusterSummary.get_supervisors();
        Collections.sort(supervisorSummaries, new Comparator<SupervisorSummary>() {

            @Override
            public int compare(SupervisorSummary o1, SupervisorSummary o2) {
                int o1Left = o1.get_numWorkers() - o1.get_numUsedWorkers();
                int o2Left = o2.get_numWorkers() - o2.get_numUsedWorkers();
                return o1Left - o2Left;
            }

        });

        for (SupervisorSummary supervisorSummary : supervisorSummaries) {
            hosts.add(supervisorSummary.get_host());
        }
        return hosts;
    } catch (Exception e) {
        if (client != null) {
            client.close();
            client = null;
        }

        LOG.error("Failed to kill all topologies ", e);
        throw new RuntimeException(e);
    }

}
 

import backtype.storm.generated.SupervisorSummary; //导入依赖的package包/类
public static SupervisorSummary mkSupervisorSummary(SupervisorInfo supervisorInfo, String supervisorId,
                                                    Map<String, Integer> supervisorToUsedSlotNum) {
    Integer usedNum = supervisorToUsedSlotNum.get(supervisorId);

    SupervisorSummary summary = new SupervisorSummary(supervisorInfo.getHostName(), supervisorId,
            supervisorInfo.getUptimeSecs(), supervisorInfo.getWorkerPorts().size(), usedNum == null ? 0 : usedNum);
    summary.set_version(supervisorInfo.getVersion());
    summary.set_buildTs(supervisorInfo.getBuildTs());
    summary.set_port(supervisorInfo.getPort() != null ? supervisorInfo.getPort() : 0);
    summary.set_errorMessage(supervisorInfo.getErrorMessage());

    return summary;
}
 

import backtype.storm.generated.SupervisorSummary; //导入依赖的package包/类
public static NimbusSummary getNimbusSummary(StormClusterState stormClusterState,
                                             List<SupervisorSummary> supervisorSummaries, NimbusData data)
        throws Exception {
    NimbusSummary ret = new NimbusSummary();

    String master = stormClusterState.get_leader_host();
    NimbusStat nimbusMaster = new NimbusStat();
    nimbusMaster.set_host(master);
    nimbusMaster.set_uptimeSecs(String.valueOf(data.uptime()));
    ret.set_nimbusMaster(nimbusMaster);

    List<NimbusStat> nimbusSlaveList = new ArrayList<>();
    ret.set_nimbusSlaves(nimbusSlaveList);
    Map<String, String> nimbusSlaveMap = Cluster.get_all_nimbus_slave(stormClusterState);
    if (nimbusSlaveMap != null) {
        for (Entry<String, String> entry : nimbusSlaveMap.entrySet()) {
            NimbusStat slave = new NimbusStat();
            slave.set_host(entry.getKey());
            slave.set_uptimeSecs(entry.getValue());

            nimbusSlaveList.add(slave);
        }
    }

    int totalPort = 0;
    int usedPort = 0;

    for (SupervisorSummary supervisor : supervisorSummaries) {
        totalPort += supervisor.get_numWorkers();
        usedPort += supervisor.get_numUsedWorkers();
    }

    ret.set_supervisorNum(supervisorSummaries.size());
    ret.set_totalPortNum(totalPort);
    ret.set_usedPortNum(usedPort);
    ret.set_freePortNum(totalPort - usedPort);
    ret.set_version(Utils.getVersion());

    return ret;
}
 

import backtype.storm.generated.SupervisorSummary; //导入依赖的package包/类
/**
 * Connvert thrift ClusterSummary to UI bean ClusterSumm
 * 
 * @param summ
 * @return
 */
public static List<ClusterSumm> clusterSummary(ClusterSummary summ,
		NimbusClient client, Map conf) throws Exception {
	// "Supervisors" "Used slots" "Free slots" "Total slots" "Running task"
	List<SupervisorSummary> sups = summ.get_supervisors();
	int supSize = 0;

	int totalMemSlots = 0;
	int useMemSlots = 0;
	int freeMemSlots = 0;

	int totalPortSlots = 0;
	int usePortSlots = 0;
	int freePortSlots = 0;

	if (sups != null) {
		supSize = sups.size();
		for (SupervisorSummary ss : sups) {

			totalPortSlots += ss.get_num_workers();
			usePortSlots += ss.get_num_used_workers();
		}

		freeMemSlots = totalMemSlots - useMemSlots;
		freePortSlots = totalPortSlots - usePortSlots;
	}

	// "Running tasks"
	int totalTasks = 0;
	List<TopologySummary> topos = summ.get_topologies();
	if (topos != null) {
		int topoSize = topos.size();
		for (int j = 0; j < topoSize; j++) {
			totalTasks += topos.get(j).get_num_tasks();
		}

	}

	String nimbustime = StatBuckets.prettyUptimeStr(summ
			.get_nimbus_uptime_secs());

	List<ClusterSumm> clusumms = new ArrayList<ClusterSumm>();

	ClusterSumm clusterSumm = new ClusterSumm();
	String master = client.getMasterHost();
	
	if (master.contains(":")) {
		String firstPart = master.substring(0, master.indexOf(":") );
		String lastPart = master.substring(master.indexOf(":"));
		clusterSumm.setNimbusHostname(NetWorkUtils.ip2Host(firstPart) + lastPart);
		clusterSumm.setNimbusIp(NetWorkUtils.host2Ip(firstPart));
	} else {
		clusterSumm.setNimbusHostname(master);
		clusterSumm.setNimbusIp(NetWorkUtils.host2Ip(master));
	}
	int port = ConfigExtension.getNimbusDeamonHttpserverPort(conf);
	clusterSumm.setNimbusLogPort(String.valueOf(port));
	clusterSumm.setNimbusUptime(nimbustime);
	clusterSumm.setSupervisorNum(String.valueOf(supSize));
	clusterSumm.setRunningTaskNum(String.valueOf(totalTasks));

	clusterSumm.setTotalPortSlotNum(String.valueOf(totalPortSlots));
	clusterSumm.setUsedPortSlotNum(String.valueOf(usePortSlots));
	clusterSumm.setFreePortSlotNum(String.valueOf(freePortSlots));
	
	clusterSumm.setVersion(summ.get_version());

	clusumms.add(clusterSumm);
	return clusumms;
}
 

import backtype.storm.generated.SupervisorSummary; //导入依赖的package包/类
/**
 * Connvert thrift ClusterSummary to UI bean ClusterSumm
 * 
 * @param summ
 * @return
 */
public static List<ClusterSumm> clusterSummary(ClusterSummary summ,
		NimbusClient client, Map conf) throws Exception {
	// "Supervisors" "Used slots" "Free slots" "Total slots" "Running task"
	List<SupervisorSummary> sups = summ.get_supervisors();
	int supSize = 0;

	int totalMemSlots = 0;
	int useMemSlots = 0;
	int freeMemSlots = 0;

	int totalPortSlots = 0;
	int usePortSlots = 0;
	int freePortSlots = 0;

	if (sups != null) {
		supSize = sups.size();
		for (SupervisorSummary ss : sups) {

			totalPortSlots += ss.get_num_workers();
			usePortSlots += ss.get_num_used_workers();
		}

		freeMemSlots = totalMemSlots - useMemSlots;
		freePortSlots = totalPortSlots - usePortSlots;
	}

	// "Running tasks"
	int totalTasks = 0;
	List<TopologySummary> topos = summ.get_topologies();
	if (topos != null) {
		int topoSize = topos.size();
		for (int j = 0; j < topoSize; j++) {
			totalTasks += topos.get(j).get_num_tasks();
		}

	}

	String nimbustime = StatBuckets.prettyUptimeStr(summ
			.get_nimbus_uptime_secs());

	List<ClusterSumm> clusumms = new ArrayList<ClusterSumm>();

	ClusterSumm clusterSumm = new ClusterSumm();
	String master = client.getMasterHost();
	clusterSumm.setNimbusHostname(master);
	if (master.contains(":")) {
		clusterSumm.setNimbusIp(NetWorkUtils.host2Ip(master.substring(0,
				master.indexOf(":"))));
	} else {
		clusterSumm.setNimbusIp(NetWorkUtils.host2Ip(master));
	}
	int port = ConfigExtension.getNimbusDeamonHttpserverPort(conf);
	clusterSumm.setNimbusLogPort(String.valueOf(port));
	clusterSumm.setNimbusUptime(nimbustime);
	clusterSumm.setSupervisorNum(String.valueOf(supSize));
	clusterSumm.setRunningTaskNum(String.valueOf(totalTasks));

	clusterSumm.setTotalPortSlotNum(String.valueOf(totalPortSlots));
	clusterSumm.setUsedPortSlotNum(String.valueOf(usePortSlots));
	clusterSumm.setFreePortSlotNum(String.valueOf(freePortSlots));
	
	clusterSumm.setVersion(summ.get_version());

	clusumms.add(clusterSumm);
	return clusumms;
}
 

import backtype.storm.generated.SupervisorSummary; //导入依赖的package包/类
public boolean metrics(Nimbus.Client client, int size, long now, MetricsState state, String message) throws Exception {
  ClusterSummary summary = client.getClusterInfo();
  long time = now - state.lastTime;
  state.lastTime = now;
  int numSupervisors = summary.get_supervisors_size();
  int totalSlots = 0;
  int totalUsedSlots = 0;
  for (SupervisorSummary sup: summary.get_supervisors()) {
    totalSlots += sup.get_num_workers();
    totalUsedSlots += sup.get_num_used_workers();
  }
  int slotsUsedDiff = totalUsedSlots - state.slotsUsed;
  state.slotsUsed = totalUsedSlots;

  int numTopologies = summary.get_topologies_size();
  long totalTransferred = 0;
  int totalExecutors = 0;
  int executorsWithMetrics = 0;
  int totalFailed = 0;
  for (TopologySummary ts: summary.get_topologies()) {
    String id = ts.get_id();
    TopologyInfo info = client.getTopologyInfo(id);
    for (ExecutorSummary es: info.get_executors()) {
      ExecutorStats stats = es.get_stats();
      totalExecutors++;
      if (stats != null) {
        if (stats.get_specific().is_set_spout()) {
          SpoutStats ss = stats.get_specific().get_spout();
          Map<String, Long> failedMap = ss.get_failed().get(":all-time");
          if (failedMap != null) {
            for (String key: failedMap.keySet()) {
              Long tmp = failedMap.get(key);
              if (tmp != null) {
                totalFailed += tmp;
              }
            }
          }
        }

        Map<String,Map<String,Long>> transferred = stats.get_transferred();
        if ( transferred != null) {
          Map<String, Long> e2 = transferred.get(":all-time");
          if (e2 != null) {
            executorsWithMetrics++;
            //The SOL messages are always on the default stream, so just count those
            Long dflt = e2.get("default");
            if (dflt != null) {
              totalTransferred += dflt;
            }
          }
        }
      }
    }
  }
  long transferredDiff = totalTransferred - state.transferred;
  state.transferred = totalTransferred;
  double throughput = (transferredDiff == 0 || time == 0) ? 0.0 : (transferredDiff * size)/(1024.0 * 1024.0)/(time/1000.0);
  System.out.println(message+"\t"+numTopologies+"\t"+totalSlots+"\t"+totalUsedSlots+"\t"+totalExecutors+"\t"+executorsWithMetrics+"\t"+now+"\t"+time+"\t"+transferredDiff+"\t"+throughput+"\t"+totalFailed);
  if ("WAITING".equals(message)) {
    //System.err.println(" !("+totalUsedSlots+" > 0 && "+slotsUsedDiff+" == 0 && "+totalExecutors+" > 0 && "+executorsWithMetrics+" >= "+totalExecutors+")");
  }
  return !(totalUsedSlots > 0 && slotsUsedDiff == 0 && totalExecutors > 0 && executorsWithMetrics >= totalExecutors);
}
 

import backtype.storm.generated.SupervisorSummary; //导入依赖的package包/类
public SupervisorEntity(SupervisorSummary supervisor){
    this(supervisor.get_supervisorId(), supervisor.get_host(), supervisor.get_uptimeSecs(),
            supervisor.get_numWorkers(), supervisor.get_numUsedWorkers());
}