Java TridentTopology类代码示例

import storm.trident.TridentTopology; //导入依赖的package包/类
public static StormTopology buildTopology()

{
	TridentTopology topology = new TridentTopology();
	RandomWordSpout spout1 = new RandomWordSpout();
	
	Stream inputStream = topology.newStream("faltu", spout1);//faltu isnt used anywhere.
	
	/**
	 * partitionPersist : The partitionPersist operation updates a source of state.
	 * It returns a TridentState object. You could then use this state in stateQuery operations elsewhere in the topology.
	 * Args:
	 * StateFactory instance - This factory implement the makeState API, that should return a instance of State.
	 * Fields list, that needs to be persisted. These field list should be present in the input stream.
	 * StateUpdater instance - The StateUpdater instance will update the underlying State.
	 */
	 inputStream
	    .partitionPersist(new RedisStoreStateFactory(), new Fields("randomWord"), new RedisStoreStateUpdater());
	 
	 return topology.build();
}
 

import storm.trident.TridentTopology; //导入依赖的package包/类
public static StormTopology buildTopology()

{
	TridentTopology topology = new TridentTopology();
	RandomPhraseSpout spout1 = new RandomPhraseSpout();
	
	Stream inputStream = topology.newStream("dumbo", spout1).parallelismHint(20);//where is dumbo used ? No where as per as I see.
	
	/**
	 * persistentAggregate : The persistentAggregate operation updates a source of state.
	 * persistentAggregate is an additional abstraction built on top of partitionPersist that knows how to take a 
	 * Trident aggregator and use it to apply updates to the source of state.
	 * Args:
	 * StateFactory instance - This factory implement the makeState API, that should return a instance of State.
	 * Fields list, that needs to be persisted. These field list should be present in the input stream.
	 * StateUpdater instance - The StateUpdater instance will update the underlying State.
	 */
	 inputStream
	    //input stream generated by spout1 has a field called randomPhrase.
	    //RandomPhraseSplitter takes a randomPhrase and additionally emits a field called randomWord into the stream.
	    .each(new Fields("randomPhrase"), new RandomPhraseSplitter(), new Fields("randomWord")).parallelismHint(6);
	    
	 return topology.build();
}
 

import storm.trident.TridentTopology; //导入依赖的package包/类
public static StormTopology buildTopology(LocalDRPC drpc) {
  FixedBatchSpout spout = new FixedBatchSpout(new Fields("sentence"), 3, new Values("the cow jumped over the moon"),
      new Values("the man went to the store and bought some candy"), new Values("four score and seven years ago"),
      new Values("how many apples can you eat"), new Values("to be or not to be the person"));
  spout.setCycle(true);

  TridentTopology topology = new TridentTopology();
  TridentState wordCounts = topology.newStream("spout1", spout).parallelismHint(16).each(new Fields("sentence"),
      new Split(), new Fields("word")).groupBy(new Fields("word")).persistentAggregate(new MemoryMapState.Factory(),
      new Count(), new Fields("count")).parallelismHint(16);

  topology.newDRPCStream("words", drpc).each(new Fields("args"), new Split(), new Fields("word")).groupBy(new Fields(
      "word")).stateQuery(wordCounts, new Fields("word"), new MapGet(), new Fields("count")).each(new Fields("count"),
      new FilterNull()).aggregate(new Fields("count"), new Sum(), new Fields("sum"));
  return topology.build();
}
 

import storm.trident.TridentTopology; //导入依赖的package包/类
public static StormTopology buildTopology(LocalDRPC drpc) {
	FixedBatchSpout spout = new FixedBatchSpout(new Fields("sentence"), 3,
			new Values("the$$cow$$jumped$$over$$the$$moon"),
			new Values("the$$man$$went$$to$$the$$store$$and$$bought$$some$$candy"),
			new Values("four$$score$$and$$seven$$years$$ago"),
			new Values("how$$many$$apples$$can$$you$$eat"),
			new Values("to$$be$$or$$not$$to$$be$$the$$person"));
	spout.setCycle(true);

	TridentTopology topology = new TridentTopology();

	TridentState wordCounts = topology.newStream("spout1", spout)
			.each(new Fields("sentence"), new Split(), new Fields("word"))
			.groupBy(new Fields("word"))
			.persistentAggregate(new MemoryMapState.Factory(), new Count(), new Fields("count"))
			.parallelismHint(6);

	topology.newDRPCStream("words", drpc).each(new Fields("args"), new Split(), new Fields("word"))
			.groupBy(new Fields("word"))
			.stateQuery(wordCounts, new Fields("word"), new MapGet(), new Fields("count"))
			.each(new Fields("count"), new FilterNull())
			.aggregate(new Fields("count"), new Sum(), new Fields("sum"));

	return topology.build();
}
 

import storm.trident.TridentTopology; //导入依赖的package包/类
public static StormTopology buildTopology(Map config) {
    FixedBatchSpout spout = new FixedBatchSpout(new Fields("sentence", "key"), 1000, new Values("the cow jumped over the moon", 1l),
            new Values("the man went to the store and bought some candy", 2l), new Values("four score and seven years ago", 3l),
            new Values("how many apples can you eat", 4l), new Values("to be or not to be the person", 5l));
    spout.setCycle(true);

    TridentTopology topology = new TridentTopology();
    Stream stream = topology.newStream("spout1", spout);

    Fields fields = new Fields("sentence", "key");
    config.put(PREFIX, "trident");
    config.put(EXTENSION, ".txt");
    config.put(PATH, "trident");
    config.put(OUTPUT_FIELDS, Arrays.asList("sentence"));
    config.put(ROTATION_SIZE, 10.0F);
    config.put(ROTATION_UNIT, "KB");
    config.put(CONTENT_TYPE, "text/plain");

    StateFactory factory = new S3StateFactory();
    stream.partitionPersist(factory, fields, new S3Updater(), new Fields()).parallelismHint(3);

    return topology.build();
}
 

import storm.trident.TridentTopology; //导入依赖的package包/类
public static StormTopology buildTopology(Config conf) {
    ITridentSpout<Object> spout;
    if (!ConfigUtil.getBoolean(conf, "spout.redis", false)) {
        spout = new OneSentencePerBatchSpout();
    } else {
        String host = (String) conf.get("redis.host");
        int port = ((Number) conf.get("redis.port")).intValue();
        String queue = (String) conf.get("redis.queue");
        spout = null;
    }
    TridentTopology topology = new TridentTopology();
    topology.newStream("spout", spout).parallelismHint(ConfigUtil.getInt(conf, "spout.parallelism", 1))
            .each(new Fields("sentence"), new Split(), new Fields("word"))
            .parallelismHint(ConfigUtil.getInt(conf, "split.parallelism", 1))
            .groupBy(new Fields("word"))
            .persistentAggregate(new MemoryMapState.Factory(), new Count(), new Fields("count"))
            .parallelismHint(ConfigUtil.getInt(conf, "counter.parallelism", 1));
    return topology.build();
}
 

import storm.trident.TridentTopology; //导入依赖的package包/类
public static StormTopology buildTopology(Config conf) {
    IRichSpout spout;
    if (!ConfigUtil.getBoolean(conf, "spout.redis", false)) {
        spout = new RandomSentenceSpout();
    } else {
        String host = (String) conf.get("redis.host");
        int port = ((Number) conf.get("redis.port")).intValue();
        String queue = (String) conf.get("redis.queue");
        spout = new RedisSentenceSpout(host, port, queue);
    }
    TridentTopology topology = new TridentTopology();
    topology.newStream("spout", spout).parallelismHint(ConfigUtil.getInt(conf, "spout.parallelism", 1))
            .each(new Fields("sentence"), new Split(), new Fields("word"))
            .parallelismHint(ConfigUtil.getInt(conf, "split.parallelism", 1))
            .groupBy(new Fields("word"))
            .persistentAggregate(new MemoryMapState.Factory(), new Count(), new Fields("count"))
            .parallelismHint(ConfigUtil.getInt(conf, "counter.parallelism", 1));
    return topology.build();
}
 

import storm.trident.TridentTopology; //导入依赖的package包/类
@Override
protected StormTopology buildTopology() {
    FixedBatchSpout spout = new FixedBatchSpout(new Fields("sentence"), 3,
            new Values("the cow jumped over the moon"),
            new Values("the man went to the store and bought some candy"),
            new Values("four score and seven years ago"),
            new Values("how many apples can you eat"),
            new Values("to be or not to be the person"));
    spout.setCycle(true);

    ESIndexState.Factory<Tweet> factory = new ESIndexState.Factory<>(getLocalClient(), Tweet.class);
    TridentTopology topology = new TridentTopology();

    TridentState state = topology.newStream("tweets", spout)
            .partitionPersist(factory, new Fields("sentence"), new ESIndexUpdater(new MyTridentTupleMapper()));

    topology.newDRPCStream("search", drpc)
            .each(new Fields("args"), new ExtractSearchArgs(), new Fields("query", "indices", "types"))
            .groupBy(new Fields("query", "indices", "types"))
            .stateQuery(state, new Fields("query", "indices", "types"), new QuerySearchIndexQuery(), new Fields("tweet"))
            .each(new Fields("tweet"), new FilterNull())
            .each(new Fields("tweet"), new CreateJson(), new Fields("json"))
            .project(new Fields("json"));

    return topology.build();
}
 

import storm.trident.TridentTopology; //导入依赖的package包/类
public static StormTopology buildTopology() {
    LOG.info("Building topology.");
    TridentTopology topology = new TridentTopology();

    GameState exampleRecursiveState = GameState.playAtRandom(new Board(), "X");
    LOG.info("SIMULATED LEAF NODE : [" + exampleRecursiveState.getBoard() + "] w/ state [" + exampleRecursiveState + "]");

    // Scoring Queue / Spout
    LocalQueueEmitter<GameState> scoringSpoutEmitter = new LocalQueueEmitter<GameState>("ScoringQueue");
    scoringSpoutEmitter.enqueue(exampleRecursiveState);
    LocalQueueSpout<GameState> scoringSpout = new LocalQueueSpout<GameState>(scoringSpoutEmitter);

    Stream inputStream = topology.newStream("scoring", scoringSpout);

    inputStream.each(new Fields("gamestate"), new isEndGame())
            .each(new Fields("gamestate"),
                    new ScoreFunction(),
                    new Fields("board", "score", "player"))
            .each(new Fields("board", "score", "player"), new ScoreUpdater(), new Fields());
    return topology.build();
}
 

import storm.trident.TridentTopology; //导入依赖的package包/类
public static void main(String[] args) throws Exception {
    final LocalCluster cluster = new LocalCluster();
    final Config conf = new Config();

    LocalDRPC client = new LocalDRPC();
    TridentTopology drpcTopology = new TridentTopology();

    drpcTopology.newDRPCStream("drpc", client)
            .each(new Fields("args"), new ArgsFunction(), new Fields("gamestate"))
            .each(new Fields("gamestate"), new GenerateBoards(), new Fields("children"))
            .each(new Fields("children"), new ScoreFunction(), new Fields("board", "score", "player"))
            .groupBy(new Fields("gamestate"))
            .aggregate(new Fields("board", "score"), new FindBestMove(), new Fields("bestMove"))
            .project(new Fields("bestMove"));

    cluster.submitTopology("drpcTopology", conf, drpcTopology.build());

    Board board = new Board();
    board.board[1][1] = "O";
    board.board[2][2] = "X";
    board.board[0][1] = "O";
    board.board[0][0] = "X";
    LOG.info("Determing best move for O on:" + board.toString());
    LOG.info("RECEIVED RESPONSE [" + client.execute("drpc", board.toKey()) + "]");
}
 

import storm.trident.TridentTopology; //导入依赖的package包/类
public static StormTopology buildTopology() {
    LOG.info("Building topology.");
    TridentTopology topology = new TridentTopology();
    StateFactory clickThruMemory = new MemoryMapState.Factory();
    ClickThruSpout spout = new ClickThruSpout();
    Stream inputStream = topology.newStream("clithru", spout);
    TridentState clickThruState = inputStream.each(new Fields("username", "campaign", "product", "click"), new Filter("click", "true"))
            .each(new Fields("username", "campaign", "product", "click"), new Distinct())
            .groupBy(new Fields("campaign"))
            .persistentAggregate(clickThruMemory, new Count(), new Fields("click_thru_count"));

    inputStream.groupBy(new Fields("campaign"))
            .persistentAggregate(new MemoryMapState.Factory(), new Count(), new Fields("impression_count"))
            .newValuesStream()
            .stateQuery(clickThruState, new Fields("campaign"), new MapGet(), new Fields("click_thru_count"))
            .each(new Fields("campaign", "impression_count", "click_thru_count"), new CampaignEffectiveness(), new Fields(""));

    return topology.build();
}
 

import storm.trident.TridentTopology; //导入依赖的package包/类
@Override
  public StormTopology getTopology(Config config) {
    final int spoutNum = BenchmarkUtils.getInt(config, SPOUT_NUM, DEFAULT_SPOUT_NUM);
    final int splitNum = BenchmarkUtils.getInt(config, SPLIT_NUM, DEFAULT_SPLIT_BOLT_NUM);
    final int countNum = BenchmarkUtils.getInt(config, COUNT_NUM, DEFAULT_COUNT_BOLT_NUM);

    spout  = new TransactionalTridentKafkaSpout(
            KafkaUtils.getTridentKafkaConfig(config, new SchemeAsMultiScheme(new StringScheme())));

    TridentTopology trident = new TridentTopology();

    trident.newStream("wordcount", spout).name("sentence").parallelismHint(spoutNum).shuffle()
            .each(new Fields(StringScheme.STRING_SCHEME_KEY), new WordSplit(), new Fields("word"))
            .parallelismHint(splitNum)
            .groupBy(new Fields("word"))
            .persistentAggregate(new MemoryMapState.Factory(), new Count(), new Fields("count"))
            .parallelismHint(countNum);
/*    trident.newStream("wordcount", spout)
      .each(new Fields(StringScheme.STRING_SCHEME_KEY), new WordSplit(), new Fields("word"))
      .groupBy(new Fields("word"))
      .persistentAggregate(new MemoryMapState.Factory(), new Count(), new Fields("count"));*/


    return trident.build();
  }
 

import storm.trident.TridentTopology; //导入依赖的package包/类
/**
	 * Topology builder
	 * @return
	 */
	public static StormTopology buildTopology() {

		TridentTopology topology = new TridentTopology();

		TridentState urlsState = topology.newStaticState(new UrlsDBFactory());
		//define the stream
		topology.newStream("countStream", new Spout())
		.each(new Fields(Spout.CLICK), new LongUrlEmitter(), new Fields(LongUrlEmitter.URL))
//			.parallelismHint(5)
		.partitionBy(new Fields(LongUrlEmitter.URL))
		.partitionPersist(new UrlsDBFactory(), new Fields(LongUrlEmitter.URL), new ProbCountBolt(), 
				new Fields(LongUrlEmitter.URL, ProbCountBolt.CURR_URL_COUNT))
//				.parallelismHint(5)
				.newValuesStream()
				.shuffle()
				.aggregate(new Fields(LongUrlEmitter.URL, ProbCountBolt.CURR_URL_COUNT), 
						new FirstNAggregator(1, ProbCountBolt.CURR_URL_COUNT, true), 
						new Fields(LongUrlEmitter.URL, ProbCountBolt.CURR_URL_COUNT))
				.each(new Fields(LongUrlEmitter.URL, ProbCountBolt.CURR_URL_COUNT), new Debug());

		return topology.build();
	}
 

import storm.trident.TridentTopology; //导入依赖的package包/类
public static StormTopology buildTopology(String hdfsUrl) {
    TridentKafkaConfig tridentKafkaConfig = new TridentKafkaConfig(new ZkHosts(ZKHOST, "/brokers"), KAFKA_TOPIC);
    tridentKafkaConfig.scheme = new SchemeAsMultiScheme(new RawScheme());
    tridentKafkaConfig.startOffsetTime = -1; // forceStartOffsetTime(-1); //Read latest messages from Kafka

    TransactionalTridentKafkaSpout tridentKafkaSpout = new TransactionalTridentKafkaSpout(tridentKafkaConfig);

    TridentTopology topology = new TridentTopology();

    Stream stream = topology.newStream("stream", tridentKafkaSpout);

    FileNameFormat fileNameFormat = new DefaultFileNameFormat().withPath(HDFS_OUT_PATH).withPrefix("trident").withExtension(".txt");
    FileRotationPolicy rotationPolicy = new FileSizeCountRotationPolicy(5.0f, FileSizeRotationPolicy.Units.MB, 10);
    HdfsState.Options seqOpts = new HdfsState.HdfsFileOptions().withFileNameFormat(fileNameFormat)
            .withRecordFormat(new DelimitedRecordFormat().withFieldDelimiter("|").withFields(new Fields("json")))
            .withRotationPolicy(rotationPolicy).withFsUrl(hdfsUrl)
            // .addRotationAction(new MoveFileAction().toDestination(HDFS_ROTATE_PATH));
            // .addRotationAction(new AddSuffixFileAction().withSuffix("-processed"));
            .addRotationAction(new MD5FileAction());
    StateFactory factory = new HdfsStateFactory().withOptions(seqOpts);

    stream.each(new Fields("bytes"), new JacksonJsonParser(), new Fields("json")).partitionPersist(factory, new Fields("json"),
            new HdfsUpdater(), new Fields());

    return topology.build();
}
 

import storm.trident.TridentTopology; //导入依赖的package包/类
public static StormTopology buildTopology(TransactionalTridentKafkaSpout spout) throws IOException {

        TridentTopology topology = new TridentTopology();
        TridentState count =
        topology
                .newStream("tweets", spout)
                .each(new Fields("str"), new ParseTweet(), new Fields("text", "content", "user"))
                .project(new Fields("content", "user"))
                .each(new Fields("content"), new OnlyHashtags())
                .each(new Fields("user"), new OnlyEnglish())
                .each(new Fields("content", "user"), new ExtractFollowerClassAndContentName(), new Fields("followerClass", "contentName"))
                .groupBy(new Fields("followerClass", "contentName"))
                .persistentAggregate(new MemoryMapState.Factory(), new Count(), new Fields("count"))
        ;


        topology
                .newDRPCStream("hashtag_count")
                .stateQuery(count, new TupleCollectionGet(), new Fields("followerClass", "contentName"))
                .stateQuery(count, new Fields("followerClass", "contentName"), new MapGet(), new Fields("count"))
                .groupBy(new Fields("followerClass"))
                .aggregate(new Fields("contentName", "count"), new FirstN.FirstNSortedAgg(1,"count", true), new Fields("contentName", "count"))
        ;

        return topology.build();
    }