Java TridentTopology.newStream方法代码示例

import storm.trident.TridentTopology; //导入方法依赖的package包/类
public static StormTopology buildTopology()

{
	TridentTopology topology = new TridentTopology();
	RandomWordSpout spout1 = new RandomWordSpout();
	
	Stream inputStream = topology.newStream("faltu", spout1);//faltu isnt used anywhere.
	
	/**
	 * partitionPersist : The partitionPersist operation updates a source of state.
	 * It returns a TridentState object. You could then use this state in stateQuery operations elsewhere in the topology.
	 * Args:
	 * StateFactory instance - This factory implement the makeState API, that should return a instance of State.
	 * Fields list, that needs to be persisted. These field list should be present in the input stream.
	 * StateUpdater instance - The StateUpdater instance will update the underlying State.
	 */
	 inputStream
	    .partitionPersist(new RedisStoreStateFactory(), new Fields("randomWord"), new RedisStoreStateUpdater());
	 
	 return topology.build();
}
 

import storm.trident.TridentTopology; //导入方法依赖的package包/类
public static StormTopology buildTopology(Map config) {
    FixedBatchSpout spout = new FixedBatchSpout(new Fields("sentence", "key"), 1000, new Values("the cow jumped over the moon", 1l),
            new Values("the man went to the store and bought some candy", 2l), new Values("four score and seven years ago", 3l),
            new Values("how many apples can you eat", 4l), new Values("to be or not to be the person", 5l));
    spout.setCycle(true);

    TridentTopology topology = new TridentTopology();
    Stream stream = topology.newStream("spout1", spout);

    Fields fields = new Fields("sentence", "key");
    config.put(PREFIX, "trident");
    config.put(EXTENSION, ".txt");
    config.put(PATH, "trident");
    config.put(OUTPUT_FIELDS, Arrays.asList("sentence"));
    config.put(ROTATION_SIZE, 10.0F);
    config.put(ROTATION_UNIT, "KB");
    config.put(CONTENT_TYPE, "text/plain");

    StateFactory factory = new S3StateFactory();
    stream.partitionPersist(factory, fields, new S3Updater(), new Fields()).parallelismHint(3);

    return topology.build();
}
 

import storm.trident.TridentTopology; //导入方法依赖的package包/类
public static StormTopology buildTopology() {
    LOG.info("Building topology.");
    TridentTopology topology = new TridentTopology();

    GameState exampleRecursiveState = GameState.playAtRandom(new Board(), "X");
    LOG.info("SIMULATED LEAF NODE : [" + exampleRecursiveState.getBoard() + "] w/ state [" + exampleRecursiveState + "]");

    // Scoring Queue / Spout
    LocalQueueEmitter<GameState> scoringSpoutEmitter = new LocalQueueEmitter<GameState>("ScoringQueue");
    scoringSpoutEmitter.enqueue(exampleRecursiveState);
    LocalQueueSpout<GameState> scoringSpout = new LocalQueueSpout<GameState>(scoringSpoutEmitter);

    Stream inputStream = topology.newStream("scoring", scoringSpout);

    inputStream.each(new Fields("gamestate"), new isEndGame())
            .each(new Fields("gamestate"),
                    new ScoreFunction(),
                    new Fields("board", "score", "player"))
            .each(new Fields("board", "score", "player"), new ScoreUpdater(), new Fields());
    return topology.build();
}
 

import storm.trident.TridentTopology; //导入方法依赖的package包/类
public static StormTopology buildTopology(String hdfsUrl) {
    TridentKafkaConfig tridentKafkaConfig = new TridentKafkaConfig(new ZkHosts(ZKHOST, "/brokers"), KAFKA_TOPIC);
    tridentKafkaConfig.scheme = new SchemeAsMultiScheme(new RawScheme());
    tridentKafkaConfig.startOffsetTime = -1; // forceStartOffsetTime(-1); //Read latest messages from Kafka

    TransactionalTridentKafkaSpout tridentKafkaSpout = new TransactionalTridentKafkaSpout(tridentKafkaConfig);

    TridentTopology topology = new TridentTopology();

    Stream stream = topology.newStream("stream", tridentKafkaSpout);

    FileNameFormat fileNameFormat = new DefaultFileNameFormat().withPath(HDFS_OUT_PATH).withPrefix("trident").withExtension(".txt");
    FileRotationPolicy rotationPolicy = new FileSizeCountRotationPolicy(5.0f, FileSizeRotationPolicy.Units.MB, 10);
    HdfsState.Options seqOpts = new HdfsState.HdfsFileOptions().withFileNameFormat(fileNameFormat)
            .withRecordFormat(new DelimitedRecordFormat().withFieldDelimiter("|").withFields(new Fields("json")))
            .withRotationPolicy(rotationPolicy).withFsUrl(hdfsUrl)
            // .addRotationAction(new MoveFileAction().toDestination(HDFS_ROTATE_PATH));
            // .addRotationAction(new AddSuffixFileAction().withSuffix("-processed"));
            .addRotationAction(new MD5FileAction());
    StateFactory factory = new HdfsStateFactory().withOptions(seqOpts);

    stream.each(new Fields("bytes"), new JacksonJsonParser(), new Fields("json")).partitionPersist(factory, new Fields("json"),
            new HdfsUpdater(), new Fields());

    return topology.build();
}
 

import storm.trident.TridentTopology; //导入方法依赖的package包/类
private static StormTopology buildTopology(final MlStormSpout mlStormSpout,
                                           final int parallelism,
                                           final int pcaRowWidth,
                                           final int numPrincipalComponents,
                                           final FieldTemplate template) {
    final TridentTopology topology = new TridentTopology();
    final Stream sensorStream = topology.newStream(FieldTemplate.FieldConstants.PCA.PCA, mlStormSpout);
    final StateFactory pcaFactory = new WindowedPcaFactory(pcaRowWidth, numPrincipalComponents, template);

    final TridentState principalComponents =
            sensorStream
                    .partitionPersist(pcaFactory, new Fields(template.getKeyField(), template.getFeatureVectorField()), new PrincipalComponentUpdater(template))
                    .parallelismHint(parallelism);


    topology.newDRPCStream(FieldTemplate.FieldConstants.PCA.PCA_DRPC)
            .broadcast()
            .stateQuery(principalComponents, new Fields(FieldTemplate.FieldConstants.ARGS), new PrincipalComponentsQuery(), new Fields(FieldTemplate.FieldConstants.PCA.PCA_COMPONENTS))
            .project(new Fields(FieldTemplate.FieldConstants.PCA.PCA_COMPONENTS))
            .aggregate(new Fields(FieldTemplate.FieldConstants.PCA.PCA_COMPONENTS), new PrincipalComponentsAggregator(), new Fields(FieldTemplate.FieldConstants.PCA.PCA_EIGEN))
            .project(new Fields(FieldTemplate.FieldConstants.PCA.PCA_EIGEN));

    return topology.build();
}
 

import storm.trident.TridentTopology; //导入方法依赖的package包/类
public static StormTopology buildTopology() {
    LOG.info("Building topology.");
    TridentTopology topology = new TridentTopology();
    SalesSpout spout = new SalesSpout();
    Stream inputStream = topology.newStream("sales", spout);
    SalesMapper mapper = new SalesMapper();
    inputStream.partitionPersist(
            new CassandraCqlIncrementalStateFactory<String, Number>(new Sum(), mapper),
            new Fields("price", "state", "product"),
            new CassandraCqlIncrementalStateUpdater<String, Number>());
    return topology.build();
}
 

import storm.trident.TridentTopology; //导入方法依赖的package包/类
@SuppressWarnings({ "rawtypes", "unchecked" })
public static StormTopology buildTopology() {
    LOG.info("Building topology.");
    TridentTopology topology = new TridentTopology();
    SimpleUpdateSpout spout = new SimpleUpdateSpout();
    Stream inputStream = topology.newStream("test", spout);
    SimpleUpdateMapper mapper = new SimpleUpdateMapper();
    inputStream.partitionPersist(new CassandraCqlStateFactory(ConsistencyLevel.ONE), new Fields("test"), new CassandraCqlStateUpdater(mapper));
    // inputStream.each(new Fields("test"), new Debug());
    return topology.build();
}
 

import storm.trident.TridentTopology; //导入方法依赖的package包/类
public static StormTopology buildTopology()

{
	TridentTopology topology = new TridentTopology();
	RandomPhraseSpout spout1 = new RandomPhraseSpout();
	
	Stream inputStream = topology.newStream("dumbo", spout1);//where is dump used ? No where as per as I see.
	
	/**
	 * persistentAggregate : The persistentAggregate operation updates a source of state.
	 * persistentAggregate is an additional abstraction built on top of partitionPersist that knows how to take a 
	 * Trident aggregator and use it to apply updates to the source of state.
	 * Args:
	 * StateFactory instance - This factory implement the makeState API, that should return a instance of State.
	 * Fields list, that needs to be persisted. These field list should be present in the input stream.
	 * StateUpdater instance - The StateUpdater instance will update the underlying State.
	 */
	 inputStream
	    //input stream generated by spout1 has a field called randomPhrase.
	    //RandomPhraseSplitter takes a randomPhrase and additionally emits a field called randomWord into the stream.
	    .each(new Fields("randomPhrase"), new RandomPhraseSplitter(), new Fields("randomWord"))
	    //the input stream is grouped by randomWord - Isn't this same as storm field grouping ? yes , similar.
	    .groupBy(new Fields("randomWord"))
	    //count the occurence of randomWord using Count aggregrator, that will add a field called count to the stream.
	    //persist the count in Redis.
	    .persistentAggregate(new RedisStoreStateFactory(), new Count(), new Fields("count"));
	 
	 return topology.build();
}
 

import storm.trident.TridentTopology; //导入方法依赖的package包/类
public static StormTopology buildTopology()

{
	TridentTopology topology = new TridentTopology();
	RandomPhraseSpout spout1 = new RandomPhraseSpout();
	
	Stream inputStream = topology.newStream("dumbo", spout1);//where is dump used ? No where as per as I see.
	
	/**
	 * persistentAggregate : The persistentAggregate operation updates a source of state.Used for grouping operations unlike partitionPersist.
	 * persistentAggregate is an additional abstraction built on top of partitionPersist that knows how to take a 
	 * Trident aggregator and use it to apply updates to the source of state.
	 * Args:
	 * StateFactory instance - This factory implement the makeState API, that should return a instance of State.
	 * Fields list, that needs to be persisted. These field list should be present in the input stream.
	 * StateUpdater instance - The StateUpdater instance will update the underlying State.
	 */
	 inputStream
	    //input stream generated by spout1 has a field called randomPhrase.
	    //RandomPhraseSplitter takes a randomPhrase and additionally emits a field called randomWord into the stream.
	    .each(new Fields("randomPhrase"), new RandomPhraseSplitter(), new Fields("randomWord"))
	    //the input stream is grouped by randomWord - Isn't this same as storm field grouping ? yes , similar.
	    .groupBy(new Fields("randomWord"))
	    //count the occurence of randomWord using Count aggregrator, that will add a field called count to the stream.
	    //persist the count in Redis.
	    .persistentAggregate(new RedisStoreStateFactory(), new Count(), new Fields("count"));
	 
	 return topology.build();
}
 

import storm.trident.TridentTopology; //导入方法依赖的package包/类
private void setUp() {
	TridentTopology topology = new TridentTopology();

	@SuppressWarnings("unchecked")
	FixedBatchSpout spout = new FixedBatchSpout(
			new Fields("sentence"), 3,
			new Values("the cow jumped over the moon"),
			new Values("the man went to the store and bought some candy"),
			new Values("four score and seven years ago"),
			new Values("how many apples can you eat"));
	spout.setCycle(true);

	Stream stream = topology.newStream("spout1", spout);
}
 

import storm.trident.TridentTopology; //导入方法依赖的package包/类
@Override
public StormTopology buildTopology(Config topologyConf) throws Exception {
    IBatchSpout wordSpout = new FileBasedBatchSpout("words.txt", new Fields("word"), 10);

    TridentTopology topology = new TridentTopology();

    Stream wordsStream = topology.newStream("someWords", wordSpout);

    TridentKafkaStateFactory stateFactory = TridentConnectorUtil.getTridentKafkaStateFactory(TOPIC_NAME, kafkaBrokerlist, "word", "word", topologyConf);
    wordsStream.partitionPersist(stateFactory, new Fields("word"), new TridentKafkaUpdater(), new Fields()).parallelismHint(1);

    JmsStateFactory jmsStateFactory = TridentConnectorUtil.getJmsStateFactory(jmsConnectionString, JMS_QUEUE_NAME);
    wordsStream.partitionPersist(jmsStateFactory, new Fields("word"), new JmsUpdater(), new Fields()).parallelismHint(1);

    Stream kafkaStream = topology.newStream("kafkaTridentSpout",  TridentConnectorUtil.getTridentKafkaEmitter(zkConnString, TOPIC_NAME, topologyConf)).parallelismHint(1);
    Stream jmsStream = topology.newStream("jmsTridentSpout",  TridentConnectorUtil.getTridentJmsSpouts(jmsConnectionString, JMS_QUEUE_NAME, topologyConf, "words")).parallelismHint(1);

    kafkaStream = kafkaStream.global().each(new Fields("str"), new TridentWordCount(), new Fields("word","count")).parallelismHint(1);
    jmsStream = jmsStream.global().each(new Fields("words"), new TridentWordCount(), new Fields("word","count")).parallelismHint(1);

    HBaseStateFactory hBaseStateFactory = TridentConnectorUtil.getTridentHbaseFactory(hbaseUrl, TABLE_NAME, "word", COLUMN_FAMILY, Lists.newArrayList("word"),
            Lists.newArrayList("count"), topologyConf);
    TridentState tridentState = jmsStream.global().partitionPersist(hBaseStateFactory, new Fields("word", "count"), new HBaseUpdater(), new Fields()).parallelismHint(1);

    HdfsStateFactory tridentHdfsFactory = TridentConnectorUtil.getTridentHdfsFactory(hdfsUrl, HDFS_SRC_DIR, HDFS_ROTATION_DIR, "word", "count");
    kafkaStream.global().partitionPersist(tridentHdfsFactory, new Fields("word", "count"), new HdfsUpdater(), new Fields()).parallelismHint(1);

    CassandraStateFactory cassandraStateFactory = TridentConnectorUtil.getCassandraStateFactory(cassandraConnString, KEY_SPACE_NAME, "word", COLUMN_FAMILY, topologyConf);
    Map<String, Class> fieldToTypeMap = Maps.newHashMap();
    fieldToTypeMap.put("word", String.class);
    fieldToTypeMap.put("count", Long.class);
    SimpleCassandraTridentTupleMapper mapper = new SimpleCassandraTridentTupleMapper(KEY_SPACE_NAME, COLUMN_FAMILY, "word",fieldToTypeMap);
    kafkaStream.global().partitionPersist(cassandraStateFactory, new Fields("word", "count"),
            new CassandraUpdater(mapper), new Fields()).parallelismHint(1);
    return topology.build();
}
 

import storm.trident.TridentTopology; //导入方法依赖的package包/类
public static void main(String... args) throws AlreadyAliveException, InvalidTopologyException, AuthorizationException {

		// starting to build topology
		TridentTopology topology = new TridentTopology();

		// Kafka as an opaque trident spout
		OpaqueTridentKafkaSpout spout = new OpaqueTridentKafkaSpoutBuilder(Conf.zookeeper, Conf.inputTopic).build();
		Stream stream = topology.newStream(kafkaSpout, spout);

		// mapping transaction messages to pairs: (person,amount)
		Stream atomicTransactions = stream.each(strF, Functions.mapToPersonAmount, personAmountF);

		// bolt to println data
		atomicTransactions.each(personAmountF, Functions.printlnFunction, emptyF);

		// aggregating transactions and mapping to Kafka messages
		Stream transactionsGroupped = atomicTransactions.groupBy(personF)
				.persistentAggregate(new MemoryMapState.Factory(), amountF, new Sum(), sumF).newValuesStream()
				.each(personSumF, Functions.mapToKafkaMessage, keyMessageF);

		// Kafka as a bolt -- producing to outputTopic
		TridentKafkaStateFactory stateFactory = new TridentKafkaStateFactory() //
				.withKafkaTopicSelector(new DefaultTopicSelector(Conf.outputTopic)) //
				.withTridentTupleToKafkaMapper(new FieldNameBasedTupleToKafkaMapper<String, String>(key, message));
		transactionsGroupped.partitionPersist(stateFactory, keyMessageF, new TridentKafkaUpdater(), emptyF);

		// submitting topology to local cluster
		new LocalCluster().submitTopology(kafkaAccountsTopology, topologyConfig(), topology.build());

		// waiting a while, then running Kafka producer
		Sleep.seconds(5);
		KafkaProduceExample.start(20);

	}
 

import storm.trident.TridentTopology; //导入方法依赖的package包/类
public static StormTopology buildTopology() {
    LOG.info("Building topology.");
    TridentTopology topology = new TridentTopology();

    // Work Queue / Spout
    LocalQueueEmitter<GameState> workSpoutEmitter = new LocalQueueEmitter<GameState>("WorkQueue");
    LocalQueueSpout<GameState> workSpout = new LocalQueueSpout<GameState>(workSpoutEmitter);
    GameState initialState = new GameState(new Board(), new ArrayList<Board>(), "X");
    workSpoutEmitter.enqueue(initialState);

    // Scoring Queue / Spout
    LocalQueueEmitter<GameState> scoringSpoutEmitter = new LocalQueueEmitter<GameState>("ScoringQueue");

    Stream inputStream = topology.newStream("gamestate", workSpout);

    inputStream.each(new Fields("gamestate"), new isEndGame())
            .each(new Fields("gamestate"),
                    new LocalQueuerFunction<GameState>(scoringSpoutEmitter),
                    new Fields(""));

    inputStream.each(new Fields("gamestate"), new GenerateBoards(), new Fields("children"))
            .each(new Fields("children"),
                    new LocalQueuerFunction<GameState>(workSpoutEmitter),
                    new Fields());

    return topology.build();
}
 

import storm.trident.TridentTopology; //导入方法依赖的package包/类
public static StormTopology buildTopology(String hdfsUrl){
    FixedBatchSpout spout = new FixedBatchSpout(new Fields("sentence", "key"), 1000, new Values("the cow jumped over the moon", 1l),
            new Values("the man went to the store and bought some candy", 2l), new Values("four score and seven years ago", 3l),
            new Values("how many apples can you eat", 4l), new Values("to be or not to be the person", 5l));
    spout.setCycle(true);

    TridentTopology topology = new TridentTopology();
    Stream stream = topology.newStream("spout1", spout);

    Fields hdfsFields = new Fields("sentence", "key");

    FileNameFormat fileNameFormat = new DefaultFileNameFormat()
            .withPath("/trident")
            .withPrefix("trident")
            .withExtension(".seq");

    FileRotationPolicy rotationPolicy = new FileSizeRotationPolicy(5.0f, FileSizeRotationPolicy.Units.MB);

    HdfsState.Options seqOpts = new HdfsState.SequenceFileOptions()
            .withFileNameFormat(fileNameFormat)
            .withSequenceFormat(new DefaultSequenceFormat("key", "sentence"))
            .withRotationPolicy(rotationPolicy)
            .withFsUrl(hdfsUrl)
            .addRotationAction(new MoveFileAction().toDestination("/dest2/"));

    StateFactory factory = new HdfsStateFactory().withOptions(seqOpts);

    TridentState state = stream
            .partitionPersist(factory, hdfsFields, new HdfsUpdater(), new Fields());

    return topology.build();
}
 

import storm.trident.TridentTopology; //导入方法依赖的package包/类
public static StormTopology buildTopology() {
    LOG.info("Building topology.");
    TridentTopology topology = new TridentTopology();
    TwitterSpout spout = new TwitterSpout();
    Stream inputStream = topology.newStream("nlp", spout);
    try {
        inputStream.each(new Fields("tweet"), new TweetSplitterFunction(), new Fields("word"))
                .each(new Fields("searchphrase", "tweet", "word"), new WordFrequencyFunction(), new Fields("baseline"))
                .each(new Fields("searchphrase", "tweet", "word", "baseline"), new PersistenceFunction(), new Fields("none"))
                .partitionPersist(new DruidStateFactory(), new Fields("searchphrase", "tweet", "word", "baseline"), new DruidStateUpdater());
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
    return topology.build();
}