Java ClusterSummary.get_topologies方法代码示例

import backtype.storm.generated.ClusterSummary; //导入方法依赖的package包/类
private static boolean topologyNameExists(Map conf, String name) {
	NimbusClient client = NimbusClient.getConfiguredClient(conf);
	try {
		ClusterSummary summary = client.getClient().getClusterInfo();
		for (TopologySummary s : summary.get_topologies()) {
			if (s.get_name().equals(name)) {
				return true;
			}
		}
		return false;

	} catch (Exception e) {
		throw new RuntimeException(e);
	} finally {
		client.close();
	}
}
 

import backtype.storm.generated.ClusterSummary; //导入方法依赖的package包/类
/**
 * @@@ Don't be compatible with Storm
 * 
 *     Here skip the logic
 * @param client
 * @param topology
 * @return
 * @throws Exception
 */
private HashSet<String> getComponents(Nimbus.Client client, String topology) throws Exception {
    HashSet<String> components = new HashSet<String>();
    ClusterSummary clusterSummary = client.getClusterInfo();
    TopologySummary topologySummary = null;
    for (TopologySummary ts : clusterSummary.get_topologies()) {
        if (topology.equals(ts.get_name())) {
            topologySummary = ts;
            break;
        }
    }
    if (topologySummary == null) {
        throw new IllegalArgumentException("topology: " + topology + " not found");
    } else {
        String id = topologySummary.get_id();
        // GetInfoOptions getInfoOpts = new GetInfoOptions();
        // getInfoOpts.set_num_err_choice(NumErrorsChoice.NONE);
        // TopologyInfo info = client.getTopologyInfoWithOpts(id, getInfoOpts);
        // for (ExecutorSummary es: info.get_executors()) {
        // components.add(es.get_component_id());
        // }
    }
    return components;
}
 

import backtype.storm.generated.ClusterSummary; //导入方法依赖的package包/类
private static boolean topologyNameExists(Map conf, String name) {
    NimbusClient client = NimbusClient.getConfiguredClient(conf);
    try {
        ClusterSummary summary = client.getClient().getClusterInfo();
        for(TopologySummary s : summary.get_topologies()) {
            if(s.get_name().equals(name)) {  
                return true;
            } 
        }  
        return false;

    } catch(Exception e) {
        throw new RuntimeException(e);
    } finally {
        client.close();
    }
}
 

import backtype.storm.generated.ClusterSummary; //导入方法依赖的package包/类
public static void cleanCluster() {
    try {
        NimbusClient client = getNimbusClient(null);

        ClusterSummary clusterSummary = client.getClient().getClusterInfo();

        List<TopologySummary> topologySummaries = clusterSummary.get_topologies();

        KillOptions killOption = new KillOptions();
        killOption.set_wait_secs(1);
        for (TopologySummary topologySummary : topologySummaries) {
            client.getClient().killTopologyWithOpts(topologySummary.get_name(), killOption);
            LOG.info("Successfully kill " + topologySummary.get_name());
        }
    } catch (Exception e) {
        if (client != null) {
            client.close();
            client = null;

        }

        LOG.error("Failed to kill all topology ", e);
    }
}
 

import backtype.storm.generated.ClusterSummary; //导入方法依赖的package包/类
private void getClusterInfo(Client client) {
    try {
        ClusterSummary clusterSummary = client.getClusterInfo();
        List<SupervisorSummary> supervisorSummaryList = clusterSummary.get_supervisors();
        int totalWorkers = 0;
        int usedWorkers = 0;
        for(SupervisorSummary summary : supervisorSummaryList){
            totalWorkers += summary.get_num_workers() ;
            usedWorkers += summary.get_num_used_workers();
        }
        int freeWorkers = totalWorkers - usedWorkers;
        LOGGER.info("cluster totalWorkers = " + totalWorkers 
                + ", usedWorkers = " + usedWorkers 
                + ", freeWorkers  = " +  freeWorkers);
        
        HttpCatClient.sendMetric("ClusterMonitor", "freeSlots", "avg", String.valueOf(freeWorkers));
        HttpCatClient.sendMetric("ClusterMonitor", "totalSlots", "avg", String.valueOf(totalWorkers));
        
        List<TopologySummary> topologySummaryList = clusterSummary.get_topologies();
        long clusterTPS = 0l;
        for(TopologySummary topology : topologySummaryList){
            long topologyTPS = getTopologyTPS(topology, client);
            clusterTPS += topologyTPS;
            if(topology.get_name().startsWith("ClusterMonitor")){
                continue;
            }
            HttpCatClient.sendMetric(topology.get_name(), topology.get_name() + "-TPS", "avg", String.valueOf(topologyTPS));
        }
        HttpCatClient.sendMetric("ClusterMonitor", "ClusterEmitTPS", "avg", String.valueOf(clusterTPS));
        
    } catch (TException e) {
        initClient(configMap);
        LOGGER.error("get client info error.", e);
    }
    catch(NotAliveException nae){
        LOGGER.warn("topology is dead.", nae);
    }
}
 

import backtype.storm.generated.ClusterSummary; //导入方法依赖的package包/类
/**
 * Returns whether a given topology exists within <code>summary</code>.
 * 
 * @param summary the summary object
 * @param topologyName the topology name
 * @return <code>true</code> if the topology exists, <code>false</code> else
 */
private static boolean topologyExists(ClusterSummary summary, String topologyName) {
    boolean result = false;
    for (TopologySummary s : summary.get_topologies()) {
        if (s.get_name().equals(topologyName)) {  
            result = true;
            break;
        } 
    }
    return result;
}
 

import backtype.storm.generated.ClusterSummary; //导入方法依赖的package包/类
/**
 * Gets the topology info based on the pipeline name.
 * 
 * @param pipelineName
 *            the pipeline to search for
 * @return the topology info (<b>null</b> if not found)
 */
public TopologyInfo getTopologyInfo(String pipelineName) {
    TopologyInfo result = null;
    ClusterSummary summary;
    try {
        logger.info("The thrift connection is " + connection);
        if (connection != null) {
            summary = connection.getClusterSummary();
            List<TopologySummary> topologies = summary.get_topologies();
            for (int t = 0; t < topologies.size(); t++) {
                TopologySummary topologySummary = topologies.get(t);
                if (pipelineName.equals(topologySummary.get_name())) {
                    try {
                        logger.info("Obtaining the TopologyInfo for the pipeine: " + pipelineName);
                        result = connection.getTopologyInfo(topologySummary
                            .get_id());
                        logger.info("the TopologyInfo is " + result);
                    } catch (NotAliveException | TException e) {
                    }
                }
            }
        }
    } catch (TException e1) {
        e1.printStackTrace();
    }
    return result;
}
 

import backtype.storm.generated.ClusterSummary; //导入方法依赖的package包/类
/**
 * Returns a topology summary by <code>name</code>.
 * 
 * @param name the topology name
 * @return the topology summary (may be <b>null</b>)
 * @throws TException in case of problems accessing the remote topology info
 */
public TopologySummary getTopologySummaryByName(String name) throws TException {
    TopologySummary result = null;
    ClusterSummary summary = getClusterSummary();
    List<TopologySummary> topologies = summary.get_topologies();
    for (int t = 0; null == result && t < topologies.size(); t++) {
        TopologySummary tSummary = topologies.get(t);
        if (tSummary.get_name().equals(name)) {
            result = tSummary;
        }
    }
    return result;
}
 

import backtype.storm.generated.ClusterSummary; //导入方法依赖的package包/类
public static TopologySummary getTopologySummary(ClusterSummary cs, String name) {
  for (TopologySummary ts : cs.get_topologies()) {
    if (name.equals(ts.get_name())) {
      return ts;
    }
  }
  return null;
}
 

import backtype.storm.generated.ClusterSummary; //导入方法依赖的package包/类
/**
 * Connvert thrift ClusterSummary to UI bean ClusterSumm
 * 
 * @param summ
 * @return
 */
public static List<ClusterSumm> clusterSummary(ClusterSummary summ,
		NimbusClient client, Map conf) throws Exception {
	// "Supervisors" "Used slots" "Free slots" "Total slots" "Running task"
	List<SupervisorSummary> sups = summ.get_supervisors();
	int supSize = 0;

	int totalMemSlots = 0;
	int useMemSlots = 0;
	int freeMemSlots = 0;

	int totalPortSlots = 0;
	int usePortSlots = 0;
	int freePortSlots = 0;

	if (sups != null) {
		supSize = sups.size();
		for (SupervisorSummary ss : sups) {

			totalPortSlots += ss.get_num_workers();
			usePortSlots += ss.get_num_used_workers();
		}

		freeMemSlots = totalMemSlots - useMemSlots;
		freePortSlots = totalPortSlots - usePortSlots;
	}

	// "Running tasks"
	int totalTasks = 0;
	List<TopologySummary> topos = summ.get_topologies();
	if (topos != null) {
		int topoSize = topos.size();
		for (int j = 0; j < topoSize; j++) {
			totalTasks += topos.get(j).get_num_tasks();
		}

	}

	String nimbustime = StatBuckets.prettyUptimeStr(summ
			.get_nimbus_uptime_secs());

	List<ClusterSumm> clusumms = new ArrayList<ClusterSumm>();

	ClusterSumm clusterSumm = new ClusterSumm();
	String master = client.getMasterHost();
	
	if (master.contains(":")) {
		String firstPart = master.substring(0, master.indexOf(":") );
		String lastPart = master.substring(master.indexOf(":"));
		clusterSumm.setNimbusHostname(NetWorkUtils.ip2Host(firstPart) + lastPart);
		clusterSumm.setNimbusIp(NetWorkUtils.host2Ip(firstPart));
	} else {
		clusterSumm.setNimbusHostname(master);
		clusterSumm.setNimbusIp(NetWorkUtils.host2Ip(master));
	}
	int port = ConfigExtension.getNimbusDeamonHttpserverPort(conf);
	clusterSumm.setNimbusLogPort(String.valueOf(port));
	clusterSumm.setNimbusUptime(nimbustime);
	clusterSumm.setSupervisorNum(String.valueOf(supSize));
	clusterSumm.setRunningTaskNum(String.valueOf(totalTasks));

	clusterSumm.setTotalPortSlotNum(String.valueOf(totalPortSlots));
	clusterSumm.setUsedPortSlotNum(String.valueOf(usePortSlots));
	clusterSumm.setFreePortSlotNum(String.valueOf(freePortSlots));
	
	clusterSumm.setVersion(summ.get_version());

	clusumms.add(clusterSumm);
	return clusumms;
}
 

import backtype.storm.generated.ClusterSummary; //导入方法依赖的package包/类
public void metrics(Nimbus.Client client, long now, MetricsState state) throws Exception {
    long totalStatted = 0;

    int componentParallelism = 0;
    boolean streamFound = false;
    ClusterSummary clusterSummary = client.getClusterInfo();
    TopologySummary topologySummary = null;
    for (TopologySummary ts : clusterSummary.get_topologies()) {
        if (_topology.equals(ts.get_name())) {
            topologySummary = ts;
            break;
        }
    }
    if (topologySummary == null) {
        throw new IllegalArgumentException("topology: " + _topology + " not found");
    } else {
        // String id = topologySummary.get_id();
        // GetInfoOptions getInfoOpts = new GetInfoOptions();
        // getInfoOpts.set_num_err_choice(NumErrorsChoice.NONE);
        // TopologyInfo info = client.getTopologyInfoWithOpts(id, getInfoOpts);
        // for (ExecutorSummary es: info.get_executors()) {
        // if (_component.equals(es.get_component_id())) {
        // componentParallelism ++;
        // ExecutorStats stats = es.get_stats();
        // if (stats != null) {
        // Map<String,Map<String,Long>> statted =
        // WATCH_EMITTED.equals(_watch) ? stats.get_emitted() : stats.get_transferred();
        // if ( statted != null) {
        // Map<String, Long> e2 = statted.get(":all-time");
        // if (e2 != null) {
        // Long stream = e2.get(_stream);
        // if (stream != null){
        // streamFound = true;
        // totalStatted += stream;
        // }
        // }
        // }
        // }
        // }
        // }
    }

    if (componentParallelism <= 0) {
        HashSet<String> components = getComponents(client, _topology);
        System.out.println("Available components for " + _topology + " :");
        System.out.println("------------------");
        for (String comp : components) {
            System.out.println(comp);
        }
        System.out.println("------------------");
        throw new IllegalArgumentException("component: " + _component + " not found");
    }

    if (!streamFound) {
        throw new IllegalArgumentException("stream: " + _stream + " not found");
    }
    long timeDelta = now - state.getLastTime();
    long stattedDelta = totalStatted - state.getLastStatted();
    state.setLastTime(now);
    state.setLastStatted(totalStatted);
    double throughput = (stattedDelta == 0 || timeDelta == 0) ? 0.0 : ((double) stattedDelta / (double) timeDelta);
    System.out.println(_topology + "\t" + _component + "\t" + componentParallelism + "\t" + _stream + "\t" + timeDelta + "\t" + stattedDelta + "\t"
            + throughput);
}
 

import backtype.storm.generated.ClusterSummary; //导入方法依赖的package包/类
/**
 * Connvert thrift ClusterSummary to UI bean ClusterSumm
 * 
 * @param summ
 * @return
 */
public static List<ClusterSumm> clusterSummary(ClusterSummary summ,
		NimbusClient client, Map conf) throws Exception {
	// "Supervisors" "Used slots" "Free slots" "Total slots" "Running task"
	List<SupervisorSummary> sups = summ.get_supervisors();
	int supSize = 0;

	int totalMemSlots = 0;
	int useMemSlots = 0;
	int freeMemSlots = 0;

	int totalPortSlots = 0;
	int usePortSlots = 0;
	int freePortSlots = 0;

	if (sups != null) {
		supSize = sups.size();
		for (SupervisorSummary ss : sups) {

			totalPortSlots += ss.get_num_workers();
			usePortSlots += ss.get_num_used_workers();
		}

		freeMemSlots = totalMemSlots - useMemSlots;
		freePortSlots = totalPortSlots - usePortSlots;
	}

	// "Running tasks"
	int totalTasks = 0;
	List<TopologySummary> topos = summ.get_topologies();
	if (topos != null) {
		int topoSize = topos.size();
		for (int j = 0; j < topoSize; j++) {
			totalTasks += topos.get(j).get_num_tasks();
		}

	}

	String nimbustime = StatBuckets.prettyUptimeStr(summ
			.get_nimbus_uptime_secs());

	List<ClusterSumm> clusumms = new ArrayList<ClusterSumm>();

	ClusterSumm clusterSumm = new ClusterSumm();
	String master = client.getMasterHost();
	clusterSumm.setNimbusHostname(master);
	if (master.contains(":")) {
		clusterSumm.setNimbusIp(NetWorkUtils.host2Ip(master.substring(0,
				master.indexOf(":"))));
	} else {
		clusterSumm.setNimbusIp(NetWorkUtils.host2Ip(master));
	}
	int port = ConfigExtension.getNimbusDeamonHttpserverPort(conf);
	clusterSumm.setNimbusLogPort(String.valueOf(port));
	clusterSumm.setNimbusUptime(nimbustime);
	clusterSumm.setSupervisorNum(String.valueOf(supSize));
	clusterSumm.setRunningTaskNum(String.valueOf(totalTasks));

	clusterSumm.setTotalPortSlotNum(String.valueOf(totalPortSlots));
	clusterSumm.setUsedPortSlotNum(String.valueOf(usePortSlots));
	clusterSumm.setFreePortSlotNum(String.valueOf(freePortSlots));
	
	clusterSumm.setVersion(summ.get_version());

	clusumms.add(clusterSumm);
	return clusumms;
}
 

import backtype.storm.generated.ClusterSummary; //导入方法依赖的package包/类
public boolean metrics(Nimbus.Client client, int size, long now, MetricsState state, String message) throws Exception {
  ClusterSummary summary = client.getClusterInfo();
  long time = now - state.lastTime;
  state.lastTime = now;
  int numSupervisors = summary.get_supervisors_size();
  int totalSlots = 0;
  int totalUsedSlots = 0;
  for (SupervisorSummary sup: summary.get_supervisors()) {
    totalSlots += sup.get_num_workers();
    totalUsedSlots += sup.get_num_used_workers();
  }
  int slotsUsedDiff = totalUsedSlots - state.slotsUsed;
  state.slotsUsed = totalUsedSlots;

  int numTopologies = summary.get_topologies_size();
  long totalTransferred = 0;
  int totalExecutors = 0;
  int executorsWithMetrics = 0;
  int totalFailed = 0;
  for (TopologySummary ts: summary.get_topologies()) {
    String id = ts.get_id();
    TopologyInfo info = client.getTopologyInfo(id);
    for (ExecutorSummary es: info.get_executors()) {
      ExecutorStats stats = es.get_stats();
      totalExecutors++;
      if (stats != null) {
        if (stats.get_specific().is_set_spout()) {
          SpoutStats ss = stats.get_specific().get_spout();
          Map<String, Long> failedMap = ss.get_failed().get(":all-time");
          if (failedMap != null) {
            for (String key: failedMap.keySet()) {
              Long tmp = failedMap.get(key);
              if (tmp != null) {
                totalFailed += tmp;
              }
            }
          }
        }

        Map<String,Map<String,Long>> transferred = stats.get_transferred();
        if ( transferred != null) {
          Map<String, Long> e2 = transferred.get(":all-time");
          if (e2 != null) {
            executorsWithMetrics++;
            //The SOL messages are always on the default stream, so just count those
            Long dflt = e2.get("default");
            if (dflt != null) {
              totalTransferred += dflt;
            }
          }
        }
      }
    }
  }
  long transferredDiff = totalTransferred - state.transferred;
  state.transferred = totalTransferred;
  double throughput = (transferredDiff == 0 || time == 0) ? 0.0 : (transferredDiff * size)/(1024.0 * 1024.0)/(time/1000.0);
  System.out.println(message+"\t"+numTopologies+"\t"+totalSlots+"\t"+totalUsedSlots+"\t"+totalExecutors+"\t"+executorsWithMetrics+"\t"+now+"\t"+time+"\t"+transferredDiff+"\t"+throughput+"\t"+totalFailed);
  if ("WAITING".equals(message)) {
    //System.err.println(" !("+totalUsedSlots+" > 0 && "+slotsUsedDiff+" == 0 && "+totalExecutors+" > 0 && "+executorsWithMetrics+" >= "+totalExecutors+")");
  }
  return !(totalUsedSlots > 0 && slotsUsedDiff == 0 && totalExecutors > 0 && executorsWithMetrics >= totalExecutors);
}