Java Nimbus.Client方法代码示例

import backtype.storm.generated.Nimbus; //导入方法依赖的package包/类
/**
 * @@@ Don't be compatible with Storm
 * 
 *     Here skip the logic
 * @param client
 * @param topology
 * @return
 * @throws Exception
 */
private HashSet<String> getComponents(Nimbus.Client client, String topology) throws Exception {
    HashSet<String> components = new HashSet<String>();
    ClusterSummary clusterSummary = client.getClusterInfo();
    TopologySummary topologySummary = null;
    for (TopologySummary ts : clusterSummary.get_topologies()) {
        if (topology.equals(ts.get_name())) {
            topologySummary = ts;
            break;
        }
    }
    if (topologySummary == null) {
        throw new IllegalArgumentException("topology: " + topology + " not found");
    } else {
        String id = topologySummary.get_id();
        // GetInfoOptions getInfoOpts = new GetInfoOptions();
        // getInfoOpts.set_num_err_choice(NumErrorsChoice.NONE);
        // TopologyInfo info = client.getTopologyInfoWithOpts(id, getInfoOpts);
        // for (ExecutorSummary es: info.get_executors()) {
        // components.add(es.get_component_id());
        // }
    }
    return components;
}
 

import backtype.storm.generated.Nimbus; //导入方法依赖的package包/类
private List<String> getSlots() throws Exception {
    NimbusClient nimbusClient = NimbusClient.getConfiguredClient(conf);
    Nimbus.Client nimbus = nimbusClient.getClient();
    List<String> slots = nimbus.getClusterInfo().get_supervisors().stream()
            .flatMap(s -> Arrays.asList(s.get_host() + ":6700", s.get_host() + ":6701").stream())
            .collect(Collectors.toList());
    Collections.shuffle(slots);
    System.out.println(slots.size());
    return slots;
}
 

import backtype.storm.generated.Nimbus; //导入方法依赖的package包/类
private void doRebalance(String topoName, int numWorkers) throws Exception {
    NimbusClient nimbusClient = NimbusClient.getConfiguredClient(conf);
    Nimbus.Client nimbus = nimbusClient.getClient();
    RebalanceOptions options = new RebalanceOptions();
    options.set_num_workers(numWorkers);
    options.set_wait_secs(1);
    System.out.println("Reassigning to " + numWorkers + " workers");
    nimbus.rebalance(topoName, options);
}
 

import backtype.storm.generated.Nimbus; //导入方法依赖的package包/类
public Nimbus.Client getClient() {
	return _client;
}
 

import backtype.storm.generated.Nimbus; //导入方法依赖的package包/类
@Override
protected void flush() {
	_client = new Nimbus.Client(_protocol);
}
 

import backtype.storm.generated.Nimbus; //导入方法依赖的package包/类
public Nimbus.Client getClient() {
    return _client;
}
 

import backtype.storm.generated.Nimbus; //导入方法依赖的package包/类
@Override
protected void flush() {
    // TODO Auto-generated method stub
    _client = new Nimbus.Client(_protocol);
}
 

import backtype.storm.generated.Nimbus; //导入方法依赖的package包/类
public NimbusClient(Map conf, String host, int port, Integer timeout) throws TTransportException {
    super(conf, ThriftConnectionType.NIMBUS, host, port, timeout, null);
    _client = new Nimbus.Client(_protocol);
}
 

import backtype.storm.generated.Nimbus; //导入方法依赖的package包/类
public void metrics(Nimbus.Client client) throws Exception {
    if (_interval <= 0) {
        throw new IllegalArgumentException("poll interval must be positive");
    }

    if (_topology == null || _topology.isEmpty()) {
        throw new IllegalArgumentException("topology name must be something");
    }

    if (_component == null || _component.isEmpty()) {
        HashSet<String> components = getComponents(client, _topology);
        System.out.println("Available components for " + _topology + " :");
        System.out.println("------------------");
        for (String comp : components) {
            System.out.println(comp);
        }
        System.out.println("------------------");
        System.out.println("Please use -m to specify one component");
        return;
    }

    if (_stream == null || _stream.isEmpty()) {
        throw new IllegalArgumentException("stream name must be something");
    }

    if (!WATCH_TRANSFERRED.equals(_watch) && !WATCH_EMITTED.equals(_watch)) {
        throw new IllegalArgumentException("watch item must either be transferred or emitted");
    }
    System.out.println("topology\tcomponent\tparallelism\tstream\ttime-diff ms\t" + _watch + "\tthroughput (Kt/s)");

    long pollMs = _interval * 1000;
    long now = System.currentTimeMillis();
    MetricsState state = new MetricsState(now, 0);
    Poller poller = new Poller(now, pollMs);

    do {
        metrics(client, now, state);
        try {
            now = poller.nextPoll();
        } catch (InterruptedException e) {
            e.printStackTrace();
            break;
        }
    } while (true);
}
 

import backtype.storm.generated.Nimbus; //导入方法依赖的package包/类
@Override
protected void flush() {
	// TODO Auto-generated method stub
	_client = new Nimbus.Client(_protocol);
}
 

import backtype.storm.generated.Nimbus; //导入方法依赖的package包/类
@Test
public void testRebalanceUsingTopologyHelper() throws Exception {

    conf.put(Config.NIMBUS_HOST, "192.168.0.31");
    conf.put(Config.NIMBUS_THRIFT_PORT, 6627);

    conf.put("resa.opt.smd.qos.ms", 1500.0);
    conf.put("resa.opt.win.history.size", 3);

    GeneralTopologyContext gtc = TopologyHelper.getGeneralTopologyContext("ta1wc", conf);

    if (gtc == null) {
        System.out.println("gtc is null");
        return;
    }

    String host = "192.168.0.31";
    int port = 6379;
    String queue = "ta1wc";
    int maxLen = 500;

    String topoName = "ta1wc";

    NimbusClient nimbusClient = NimbusClient.getConfiguredClient(conf);
    Nimbus.Client nimbus = nimbusClient.getClient();
    String topoId = TopologyHelper.getTopologyId(nimbus, topoName);

    Map<String, List<ExecutorDetails>> comp2Executors = TopologyHelper.getTopologyExecutors(topoName, conf)
            .entrySet().stream().filter(e -> !Utils.isSystemId(e.getKey()))
            .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));

    for (int i = 0; i < 10000; i++) {
        Utils.sleep(10000);

        TopologyInfo topoInfo = nimbus.getTopologyInfo(topoId);
        Map<String, Integer> currAllocation = topoInfo.get_executors().stream().filter(e -> !Utils.isSystemId(e.get_component_id()))
                .collect(Collectors.groupingBy(e -> e.get_component_id(),
                        Collectors.reducing(0, e -> 1, (i1, i2) -> i1 + i2)));

        System.out.println("-------------Report on: " + System.currentTimeMillis() + "------------------------------");
        System.out.println(currAllocation);
    }
}
 

import backtype.storm.generated.Nimbus; //导入方法依赖的package包/类
@Test
public void testMakeUsingTopologyHelper() throws Exception {

    conf.put(Config.NIMBUS_HOST, "192.168.0.30");
    conf.put(Config.NIMBUS_THRIFT_PORT, 6627);

    conf.put("resa.opt.smd.qos.ms", 1500.0);
    conf.put("resa.opt.win.history.size", 3);

    conf.put("resa.comp.sample.rate", 1.0);

    conf.put(ResaConfig.ALLOWED_EXECUTOR_NUM, 7);

    GeneralTopologyContext gtc = TopologyHelper.getGeneralTopologyContext("ta1wc2Redis", conf);

    if (gtc == null) {
        System.out.println("gtc is null");
        return;
    }

    String host = "192.168.0.30";
    int port = 6379;
    String queue = "ta1wc";
    int maxLen = 500;

    String topoName = "ta1wc2Redis";

    NimbusClient nimbusClient = NimbusClient.getConfiguredClient(conf);
    Nimbus.Client nimbus = nimbusClient.getClient();
    String topoId = TopologyHelper.getTopologyId(nimbus, topoName);

    TopologyInfo topoInfo = nimbus.getTopologyInfo(topoId);

    Map<String, Integer> currAllocation = topoInfo.get_executors().stream().filter(e -> !Utils.isSystemId(e.get_component_id()))
            .collect(Collectors.groupingBy(e -> e.get_component_id(),
                    Collectors.reducing(0, e -> 1, (i1, i2) -> i1 + i2)));

    SimpleGeneralAllocCalculator smdm = new SimpleGeneralAllocCalculator();
    smdm.init(conf, currAllocation, gtc.getRawTopology());

    Map<String, List<ExecutorDetails>> comp2Executors = TopologyHelper.getTopologyExecutors(topoName, conf)
            .entrySet().stream().filter(e -> !Utils.isSystemId(e.getKey()))
            .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));

    for (int i = 0; i < 10000; i++) {
        Utils.sleep(30000);

        topoInfo = nimbus.getTopologyInfo(topoId);
        Map<String, Integer> updatedAllocation = topoInfo.get_executors().stream().filter(e -> !Utils.isSystemId(e.get_component_id()))
                .collect(Collectors.groupingBy(e -> e.get_component_id(),
                        Collectors.reducing(0, e -> 1, (i1, i2) -> i1 + i2)));

        AggResultCalculator resultCalculator = new AggResultCalculator(
                RedisDataSource.readData(host, port, queue, maxLen), comp2Executors, gtc.getRawTopology());
        resultCalculator.calCMVStat();

        System.out.println("-------------Report on: " + System.currentTimeMillis() + "------------------------------");
        if (currAllocation.equals(updatedAllocation)) {
            System.out.println(currAllocation + "-->" + smdm.calc(resultCalculator.getResults(), 7));
        } else {
            currAllocation = updatedAllocation;
            smdm.allocationChanged(currAllocation);
            System.out.println("Allocation updated to " + currAllocation);
        }
    }
}
 

import backtype.storm.generated.Nimbus; //导入方法依赖的package包/类
@Test
public void testMakeUsingTopologyHelperForkTopology() throws Exception {

    conf.put(Config.NIMBUS_HOST, "192.168.0.30");
    conf.put(Config.NIMBUS_THRIFT_PORT, 6627);

    conf.put("resa.opt.smd.qos.ms", 1500.0);
    conf.put("resa.opt.win.history.size", 3);
    conf.put("resa.opt.win.history.size.ignore", -1);
    conf.put("resa.comp.sample.rate", 1.0);

    int allewedExecutorNum = 24;
    conf.put(ResaConfig.ALLOWED_EXECUTOR_NUM, allewedExecutorNum);

    String host = "192.168.0.30";
    int port = 6379;
    String queue = "ta1wc";
    int maxLen = 5000;

    NimbusClient nimbusClient = NimbusClient.getConfiguredClient(conf);
    Nimbus.Client nimbus = nimbusClient.getClient();
    //String topoName = "ta1wc2P2Redis";
    ///String topoName = "ta1wcLoopRedis";
    ///String topoName = "arwcRedis";
    ///String topoName = "outdetResa";
    ///String topoName = "rwc";
    String topoName = "fpt";
    String topoId = TopologyHelper.getTopologyId(nimbus, topoName);

    TopologyInfo topoInfo = nimbus.getTopologyInfo(topoId);

    Map<String, Integer> currAllocation = topoInfo.get_executors().stream().filter(e -> !Utils.isSystemId(e.get_component_id()))
            .collect(Collectors.groupingBy(e -> e.get_component_id(),
                    Collectors.reducing(0, e -> 1, (i1, i2) -> i1 + i2)));

    SimpleGeneralAllocCalculator smdm = new SimpleGeneralAllocCalculator();
    smdm.init(conf, currAllocation, nimbus.getUserTopology(topoId));


    for (int i = 0; i < 10000; i++) {
        Utils.sleep(30000);

        topoInfo = nimbus.getTopologyInfo(topoId);
        Map<String, Integer> updatedAllocation = topoInfo.get_executors().stream().filter(e -> !Utils.isSystemId(e.get_component_id()))
                .collect(Collectors.groupingBy(e -> e.get_component_id(),
                        Collectors.reducing(0, e -> 1, (i1, i2) -> i1 + i2)));

        Map<String, List<ExecutorDetails>> comp2Executors = TopologyHelper.getTopologyExecutors(topoName, conf)
                .entrySet().stream().filter(e -> !Utils.isSystemId(e.getKey()))
                .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));

        AggResultCalculator resultCalculator = new AggResultCalculator(
                RedisDataSource.readData(host, port, queue, maxLen), comp2Executors, nimbus.getUserTopology(topoId));
        resultCalculator.calCMVStat();

        System.out.println("-------------Report on: " + System.currentTimeMillis() + "------------------------------");
        if (currAllocation.equals(updatedAllocation)) {
            System.out.println(currAllocation + "-->" + smdm.calc(resultCalculator.getResults(), allewedExecutorNum));
        } else {
            currAllocation = updatedAllocation;
            smdm.allocationChanged(currAllocation);
            RedisDataSource.clearQueue(host, port, queue);
            System.out.println("Allocation updated to " + currAllocation);
        }
    }
}
 

import backtype.storm.generated.Nimbus; //导入方法依赖的package包/类
@Test
public void runAllocCalculatorAlg() throws Exception {

    conf.put(Config.NIMBUS_HOST, "192.168.0.30");
    conf.put(Config.NIMBUS_THRIFT_PORT, 6627);

    conf.put("resa.opt.smd.qos.ms", 1500.0);
    conf.put("resa.opt.win.history.size", 1);
    conf.put("resa.opt.win.history.size.ignore", 0);
    conf.put("resa.comp.sample.rate", 1.0);

    int allewedExecutorNum = 24;
    conf.put(ResaConfig.ALLOWED_EXECUTOR_NUM, allewedExecutorNum);

    String host = "192.168.0.30";
    int port = 6379;
    String queue = "fpm-7-1417774110-metrics";
    int maxLen = 5000;

    NimbusClient nimbusClient = NimbusClient.getConfiguredClient(conf);
    Nimbus.Client nimbus = nimbusClient.getClient();
    //String topoName = "ta1wc2P2Redis";
    ///String topoName = "ta1wcLoopRedis";
    ///String topoName = "arwcRedis";
    ///String topoName = "outdetResa";
    ///String topoName = "rwc";
    String topoName = "fpm";
    String topoId = TopologyHelper.getTopologyId(nimbus, topoName);

    TopologyInfo topoInfo = nimbus.getTopologyInfo(topoId);

    Map<String, Integer> currAllocation = topoInfo.get_executors().stream().filter(e -> !Utils.isSystemId(e.get_component_id()))
            .collect(Collectors.groupingBy(e -> e.get_component_id(),
                    Collectors.reducing(0, e -> 1, (i1, i2) -> i1 + i2)));

    SimpleGeneralAllocCalculator smdm = new SimpleGeneralAllocCalculator();
    smdm.init(conf, currAllocation, nimbus.getUserTopology(topoId));

    Map<String, List<ExecutorDetails>> comp2Executors = TopologyHelper.getTopologyExecutors(topoName, conf)
            .entrySet().stream().filter(e -> !Utils.isSystemId(e.getKey()))
            .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));

    AggResultCalculator resultCalculator = new AggResultCalculator(
            RedisDataSource.iterData(host, port, queue, maxLen), comp2Executors, nimbus.getUserTopology(topoId));
    resultCalculator.calCMVStat();

    System.out.println("-------------Report on: " + System.currentTimeMillis() + "------------------------------");
    System.out.println(currAllocation + "-->" + smdm.calc(resultCalculator.getResults(), allewedExecutorNum).currOptAllocation);
}
 

import backtype.storm.generated.Nimbus; //导入方法依赖的package包/类
public NimbusClient(Map conf, String host, int port, Integer timeout) throws TTransportException {
    super(conf, host, port, timeout);
    _client = new Nimbus.Client(_protocol);
}