Java StormTopology类代码示例

import backtype.storm.generated.StormTopology; //导入依赖的package包/类
public static StormTopology buildTopology()

{
	TridentTopology topology = new TridentTopology();
	RandomWordSpout spout1 = new RandomWordSpout();
	
	Stream inputStream = topology.newStream("faltu", spout1);//faltu isnt used anywhere.
	
	/**
	 * partitionPersist : The partitionPersist operation updates a source of state.
	 * It returns a TridentState object. You could then use this state in stateQuery operations elsewhere in the topology.
	 * Args:
	 * StateFactory instance - This factory implement the makeState API, that should return a instance of State.
	 * Fields list, that needs to be persisted. These field list should be present in the input stream.
	 * StateUpdater instance - The StateUpdater instance will update the underlying State.
	 */
	 inputStream
	    .partitionPersist(new RedisStoreStateFactory(), new Fields("randomWord"), new RedisStoreStateUpdater());
	 
	 return topology.build();
}
 

import backtype.storm.generated.StormTopology; //导入依赖的package包/类
public static StormTopology buildTopology()

{
	TridentTopology topology = new TridentTopology();
	RandomPhraseSpout spout1 = new RandomPhraseSpout();
	
	Stream inputStream = topology.newStream("dumbo", spout1).parallelismHint(20);//where is dumbo used ? No where as per as I see.
	
	/**
	 * persistentAggregate : The persistentAggregate operation updates a source of state.
	 * persistentAggregate is an additional abstraction built on top of partitionPersist that knows how to take a 
	 * Trident aggregator and use it to apply updates to the source of state.
	 * Args:
	 * StateFactory instance - This factory implement the makeState API, that should return a instance of State.
	 * Fields list, that needs to be persisted. These field list should be present in the input stream.
	 * StateUpdater instance - The StateUpdater instance will update the underlying State.
	 */
	 inputStream
	    //input stream generated by spout1 has a field called randomPhrase.
	    //RandomPhraseSplitter takes a randomPhrase and additionally emits a field called randomWord into the stream.
	    .each(new Fields("randomPhrase"), new RandomPhraseSplitter(), new Fields("randomWord")).parallelismHint(6);
	    
	 return topology.build();
}
 

import backtype.storm.generated.StormTopology; //导入依赖的package包/类
public static void main(String[] args) {

    Config config = new Config();
    config.setDebug(true);

    StormTopology topology = buildTopology();
    // Un-comment to run locally:
    LocalCluster localCluster = new LocalCluster();
    localCluster.submitTopology("local-moving-avg", config, topology);
    // Un-comment to run as part of a Storm cluster:
    // try {
    //   StormSubmitter.submitTopology("cluster-moving-average",
    //  				    config,
    // 				    topology);
    // } catch(AlreadyAliveException e) {
    //   e.printStackTrace();
    // } catch(InvalidTopologyException e) {
    //   e.printStackTrace();
    //}
  }
 

import backtype.storm.generated.StormTopology; //导入依赖的package包/类
public static void run(final String name, final StormTopology topo, final Counter hasCompleted) throws Exception {
    Thread th = new Thread(new Runnable() {
        @Override
        public void run() {
            try {
                start(name, topo);
                hasCompleted.waitForZero(TimeValue.timeValueSeconds(15));
            } finally {
                stop(name);
            }

        }
    }, "test-storm-runner");
    th.setDaemon(true);

    copyPropertiesIntoCfg(cfg);
    th.start();
}
 

import backtype.storm.generated.StormTopology; //导入依赖的package包/类
public TopologyContext(StormTopology topology, Map stormConf,
		Map<Integer, String> taskToComponent,
		Map<String, List<Integer>> componentToSortedTasks,
		Map<String, Map<String, Fields>> componentToStreamToFields,
		String topologyId, String codeDir, String pidDir, Integer taskId,
		Integer workerPort, List<Integer> workerTasks,
		Map<String, Object> defaultResources,
		Map<String, Object> userResources,
		Map<String, Object> executorData, Map registeredMetrics,
		clojure.lang.Atom openOrPrepareWasCalled) {
	super(topology, stormConf, taskToComponent, componentToSortedTasks,
			componentToStreamToFields, topologyId, codeDir, pidDir,
			workerPort, workerTasks, defaultResources, userResources);
	_taskId = taskId;
	_executorData = executorData;
	_registeredMetrics = registeredMetrics;
	_openOrPrepareWasCalled = openOrPrepareWasCalled;
}
 

import backtype.storm.generated.StormTopology; //导入依赖的package包/类
public static StormTopology add_metrics_component(StormTopology topology) {

		/**
		 */
		//		(defn metrics-consumer-bolt-specs [storm-conf topology]
		//				  (let [component-ids-that-emit-metrics (cons SYSTEM-COMPONENT-ID (keys (all-components topology)))
		//				        inputs (->> (for [comp-id component-ids-that-emit-metrics]
		//				                      {[comp-id METRICS-STREAM-ID] :shuffle})
		//				                    (into {}))
		//				        
		//				        mk-bolt-spec (fn [class arg p]
		//				                       (thrift/mk-bolt-spec*
		//				                        inputs
		//				                        (backtype.storm.metric.MetricsConsumerBolt. class arg)
		//				                        {} :p p :conf {TOPOLOGY-TASKS p}))]
		//				    
		//				    (map
		//				     (fn [component-id register]           
		//				       [component-id (mk-bolt-spec (get register "class")
		//				                                   (get register "argument")
		//				                                   (or (get register "parallelism.hint") 1))])
		//				     
		//				     (metrics-consumer-register-ids storm-conf)
		//				     (get storm-conf TOPOLOGY-METRICS-CONSUMER-REGISTER))))
		return topology;
	}
 

import backtype.storm.generated.StormTopology; //导入依赖的package包/类
@Override
public void submitTopologyWithOpts(String topologyName, Map conf,
		StormTopology topology, SubmitOptions submitOpts){
	
	if (!Utils.isValidConf(conf))
		throw new RuntimeException("Topology conf is not json-serializable");
	JStormUtils.setLocalMode(true);
	
	try {
		if (submitOpts == null) {
			state.getNimbus().submitTopology(topologyName, null,
					Utils.to_json(conf), topology);
		}else {
			state.getNimbus().submitTopologyWithOpts(topologyName, null,
					Utils.to_json(conf), topology, submitOpts);
		}
		
	} catch (Exception e) {
		
		LOG.error("Failed to submit topology " + topologyName, e);
		throw new RuntimeException(e);
	} 
}
 

import backtype.storm.generated.StormTopology; //导入依赖的package包/类
public WorkerTopologyContext(StormTopology topology, Map stormConf,
		Map<Integer, String> taskToComponent,
		Map<String, List<Integer>> componentToSortedTasks,
		Map<String, Map<String, Fields>> componentToStreamToFields,
		String topologyId, String codeDir, String pidDir, Integer workerPort,
		List<Integer> workerTasks, Map<String, Object> defaultResources,
		Map<String, Object> userResources) {
	super(topology, stormConf, taskToComponent, componentToSortedTasks,
			componentToStreamToFields, topologyId);
	_codeDir = codeDir;
	_defaultResources = defaultResources;
	_userResources = userResources;
	try {
		if (pidDir != null) {
			_pidDir = new File(pidDir).getCanonicalPath();
		} else {
			_pidDir = null;
		}
	} catch (IOException e) {
		throw new RuntimeException("Could not get canonical path for "
				+ _pidDir, e);
	}
	_workerPort = workerPort;
	_workerTasks = workerTasks;
}
 

import backtype.storm.generated.StormTopology; //导入依赖的package包/类
public static StormTopology buildTopology(String redisIp, String redisPort) {
	// topology to build
	TopologyBuilder topology = new TopologyBuilder();

	// create a spout
	WikiCrawlerSpout wikiSpout = new WikiCrawlerSpout(redisIp, redisPort);

	// create a bolt
	WikiCrawlerExplorerBolt wikiBolt = new WikiCrawlerExplorerBolt(redisIp, redisPort);

	// set up the DAG
	// this spout always takes 1 task, it is light
	topology.setSpout("wikiSpout", wikiSpout, 1)
			.setNumTasks(2)
			.setMaxSpoutPending(5);
	// this bolt uses as many executors(threads) as the cores available
	topology.setBolt("wikiBolt", wikiBolt, numCores)
			.setNumTasks(numCores * 4) // 4 task per thread
			.shuffleGrouping("wikiSpout");

	return topology.createTopology();
}
 

import backtype.storm.generated.StormTopology; //导入依赖的package包/类
public static ComponentCommon getComponentCommon(StormTopology topology,
		String componentId) {
	for (StormTopology._Fields f : StormTopology.metaDataMap.keySet()) {
		Map<String, Object> componentMap = (Map<String, Object>) topology
				.getFieldValue(f);
		if (componentMap.containsKey(componentId)) {
			Object component = componentMap.get(componentId);
			if (component instanceof Bolt) {
				return ((Bolt) component).get_common();
			}
			if (component instanceof SpoutSpec) {
				return ((SpoutSpec) component).get_common();
			}
			if (component instanceof StateSpoutSpec) {
				return ((StateSpoutSpec) component).get_common();
			}
			throw new RuntimeException(
					"Unreachable code! No get_common conversion for component "
							+ component);
		}
	}
	throw new IllegalArgumentException(
			"Could not find component common for " + componentId);
}
 

import backtype.storm.generated.StormTopology; //导入依赖的package包/类
public static StormTopology buildTopology(LocalDRPC drpc) {
  FixedBatchSpout spout = new FixedBatchSpout(new Fields("sentence"), 3, new Values("the cow jumped over the moon"),
      new Values("the man went to the store and bought some candy"), new Values("four score and seven years ago"),
      new Values("how many apples can you eat"), new Values("to be or not to be the person"));
  spout.setCycle(true);

  TridentTopology topology = new TridentTopology();
  TridentState wordCounts = topology.newStream("spout1", spout).parallelismHint(16).each(new Fields("sentence"),
      new Split(), new Fields("word")).groupBy(new Fields("word")).persistentAggregate(new MemoryMapState.Factory(),
      new Count(), new Fields("count")).parallelismHint(16);

  topology.newDRPCStream("words", drpc).each(new Fields("args"), new Split(), new Fields("word")).groupBy(new Fields(
      "word")).stateQuery(wordCounts, new Fields("word"), new MapGet(), new Fields("count")).each(new Fields("count"),
      new FilterNull()).aggregate(new Fields("count"), new Sum(), new Fields("sum"));
  return topology.build();
}
 

import backtype.storm.generated.StormTopology; //导入依赖的package包/类
public static void submitTopology(String name, Map stormConf,
		StormTopology topology, SubmitOptions opts, List<File> jarFiles)
		throws AlreadyAliveException, InvalidTopologyException {
	if (jarFiles == null) {
		jarFiles = new ArrayList<File>();
	}
	Map<String, String> jars = new HashMap<String, String>(jarFiles.size());
	List<String> names = new ArrayList<String>(jarFiles.size());
	
	for (File f : jarFiles) {
		if (!f.exists()) {
			LOG.info(f.getName() + " is not existed: "
					+ f.getAbsolutePath());
			continue;
		}
		jars.put(f.getName(), f.getAbsolutePath());
		names.add(f.getName());
	}
	LOG.info("Files: " + names + " will be loaded");
	stormConf.put(GenericOptionsParser.TOPOLOGY_LIB_PATH, jars);
	stormConf.put(GenericOptionsParser.TOPOLOGY_LIB_NAME, names);
	submitTopology(name, stormConf, topology, opts);
}
 

import backtype.storm.generated.StormTopology; //导入依赖的package包/类
public static StormTopology buildTopology(LocalDRPC drpc) {
	FixedBatchSpout spout = new FixedBatchSpout(new Fields("sentence"), 3,
			new Values("the$$cow$$jumped$$over$$the$$moon"),
			new Values("the$$man$$went$$to$$the$$store$$and$$bought$$some$$candy"),
			new Values("four$$score$$and$$seven$$years$$ago"),
			new Values("how$$many$$apples$$can$$you$$eat"),
			new Values("to$$be$$or$$not$$to$$be$$the$$person"));
	spout.setCycle(true);

	TridentTopology topology = new TridentTopology();

	TridentState wordCounts = topology.newStream("spout1", spout)
			.each(new Fields("sentence"), new Split(), new Fields("word"))
			.groupBy(new Fields("word"))
			.persistentAggregate(new MemoryMapState.Factory(), new Count(), new Fields("count"))
			.parallelismHint(6);

	topology.newDRPCStream("words", drpc).each(new Fields("args"), new Split(), new Fields("word"))
			.groupBy(new Fields("word"))
			.stateQuery(wordCounts, new Fields("word"), new MapGet(), new Fields("count"))
			.each(new Fields("count"), new FilterNull())
			.aggregate(new Fields("count"), new Sum(), new Fields("sum"));

	return topology.build();
}
 

import backtype.storm.generated.StormTopology; //导入依赖的package包/类
public WorkerTopologyContext(StormTopology topology, Map stormConf, Map<Integer, String> taskToComponent,
        Map<String, List<Integer>> componentToSortedTasks, Map<String, Map<String, Fields>> componentToStreamToFields, String stormId, String codeDir,
        String pidDir, Integer workerPort, List<Integer> workerTasks, Map<String, Object> defaultResources, Map<String, Object> userResources) {
    super(topology, stormConf, taskToComponent, componentToSortedTasks, componentToStreamToFields, stormId);
    _codeDir = codeDir;
    _defaultResources = defaultResources;
    _userResources = userResources;
    try {
        if (pidDir != null) {
            _pidDir = new File(pidDir).getCanonicalPath();
        } else {
            _pidDir = null;
        }
    } catch (IOException e) {
        throw new RuntimeException("Could not get canonical path for " + _pidDir, e);
    }
    _workerPort = workerPort;
    _workerTasks = workerTasks;
}
 

import backtype.storm.generated.StormTopology; //导入依赖的package包/类
public static void submitTopology(String name, Map stormConf,
		StormTopology topology, SubmitOptions opts, List<File> jarFiles)
		throws AlreadyAliveException, InvalidTopologyException {
	Map<String, String> jars = new HashMap<String, String>(jarFiles.size());
	List<String> names = new ArrayList<String>(jarFiles.size());
	if (jarFiles == null)
		jarFiles = new ArrayList<File>();
	for (File f : jarFiles) {
		if (!f.exists()) {
			LOG.info(f.getName() + " is not existed: "
					+ f.getAbsolutePath());
			continue;
		}
		jars.put(f.getName(), f.getAbsolutePath());
		names.add(f.getName());
	}
	LOG.info("Files: " + names + " will be loaded");
	stormConf.put(GenericOptionsParser.TOPOLOGY_LIB_PATH, jars);
	stormConf.put(GenericOptionsParser.TOPOLOGY_LIB_NAME, names);
	submitTopology(name, stormConf, topology, opts);
}