Java StateFactory类代码示例

import storm.trident.state.StateFactory; //导入依赖的package包/类
public static StateFactory getStateFactory(ConfigData config, String cacheName) throws CacheNotValidException {
    if(config.getCacheType().equals("gridgain")){
        return new GridGainFactory(cacheName, config.getEnrichs(),  config.getGridGainConfig());
    }else if(config.getCacheType().equals("riak")){
        return new RiakState.Factory<>("rbbi:" + cacheName, config.getRiakServers(), 8087, Map.class);
    } else if (config.getCacheType().equals("memcached")) {
        MemcachedState.Options memcachedOpts = new MemcachedState.Options();
        memcachedOpts.expiration = 60 * 60 * 1000;
        memcachedOpts.localCacheSize = 0;
        memcachedOpts.requestTimeoutMillis = 250;
        memcachedOpts.maxMultiGetBatchSize = 1000;
        return MemcachedState.transactional(config.getMemcachedServers(), memcachedOpts);
    } else if (config.getCacheType().equals("memory")) {
        return new MemoryMapState.Factory();
    }else {
        throw new CacheNotValidException("Not cache backend found: " + config.getCacheType());
    }
}
 

import storm.trident.state.StateFactory; //导入依赖的package包/类
public static StormTopology buildTopology(Map config) {
    FixedBatchSpout spout = new FixedBatchSpout(new Fields("sentence", "key"), 1000, new Values("the cow jumped over the moon", 1l),
            new Values("the man went to the store and bought some candy", 2l), new Values("four score and seven years ago", 3l),
            new Values("how many apples can you eat", 4l), new Values("to be or not to be the person", 5l));
    spout.setCycle(true);

    TridentTopology topology = new TridentTopology();
    Stream stream = topology.newStream("spout1", spout);

    Fields fields = new Fields("sentence", "key");
    config.put(PREFIX, "trident");
    config.put(EXTENSION, ".txt");
    config.put(PATH, "trident");
    config.put(OUTPUT_FIELDS, Arrays.asList("sentence"));
    config.put(ROTATION_SIZE, 10.0F);
    config.put(ROTATION_UNIT, "KB");
    config.put(CONTENT_TYPE, "text/plain");

    StateFactory factory = new S3StateFactory();
    stream.partitionPersist(factory, fields, new S3Updater(), new Fields()).parallelismHint(3);

    return topology.build();
}
 

import storm.trident.state.StateFactory; //导入依赖的package包/类
public static StormTopology buildTopology() {
    LOG.info("Building topology.");
    TridentTopology topology = new TridentTopology();
    StateFactory clickThruMemory = new MemoryMapState.Factory();
    ClickThruSpout spout = new ClickThruSpout();
    Stream inputStream = topology.newStream("clithru", spout);
    TridentState clickThruState = inputStream.each(new Fields("username", "campaign", "product", "click"), new Filter("click", "true"))
            .each(new Fields("username", "campaign", "product", "click"), new Distinct())
            .groupBy(new Fields("campaign"))
            .persistentAggregate(clickThruMemory, new Count(), new Fields("click_thru_count"));

    inputStream.groupBy(new Fields("campaign"))
            .persistentAggregate(new MemoryMapState.Factory(), new Count(), new Fields("impression_count"))
            .newValuesStream()
            .stateQuery(clickThruState, new Fields("campaign"), new MapGet(), new Fields("click_thru_count"))
            .each(new Fields("campaign", "impression_count", "click_thru_count"), new CampaignEffectiveness(), new Fields(""));

    return topology.build();
}
 

import storm.trident.state.StateFactory; //导入依赖的package包/类
public static StormTopology buildTopology(String hdfsUrl) {
    TridentKafkaConfig tridentKafkaConfig = new TridentKafkaConfig(new ZkHosts(ZKHOST, "/brokers"), KAFKA_TOPIC);
    tridentKafkaConfig.scheme = new SchemeAsMultiScheme(new RawScheme());
    tridentKafkaConfig.startOffsetTime = -1; // forceStartOffsetTime(-1); //Read latest messages from Kafka

    TransactionalTridentKafkaSpout tridentKafkaSpout = new TransactionalTridentKafkaSpout(tridentKafkaConfig);

    TridentTopology topology = new TridentTopology();

    Stream stream = topology.newStream("stream", tridentKafkaSpout);

    FileNameFormat fileNameFormat = new DefaultFileNameFormat().withPath(HDFS_OUT_PATH).withPrefix("trident").withExtension(".txt");
    FileRotationPolicy rotationPolicy = new FileSizeCountRotationPolicy(5.0f, FileSizeRotationPolicy.Units.MB, 10);
    HdfsState.Options seqOpts = new HdfsState.HdfsFileOptions().withFileNameFormat(fileNameFormat)
            .withRecordFormat(new DelimitedRecordFormat().withFieldDelimiter("|").withFields(new Fields("json")))
            .withRotationPolicy(rotationPolicy).withFsUrl(hdfsUrl)
            // .addRotationAction(new MoveFileAction().toDestination(HDFS_ROTATE_PATH));
            // .addRotationAction(new AddSuffixFileAction().withSuffix("-processed"));
            .addRotationAction(new MD5FileAction());
    StateFactory factory = new HdfsStateFactory().withOptions(seqOpts);

    stream.each(new Fields("bytes"), new JacksonJsonParser(), new Fields("json")).partitionPersist(factory, new Fields("json"),
            new HdfsUpdater(), new Fields());

    return topology.build();
}
 

import storm.trident.state.StateFactory; //导入依赖的package包/类
public static void main(String[] args) throws AlreadyAliveException, InvalidTopologyException {
    if (args.length < 3) {
        Logger.getAnonymousLogger().log(Level.SEVERE, "where are the commandline args? -- use args -- folder numWorkers windowSize");
        System.exit(-1);
    }

    final FieldTemplate template = new MlStormFieldTemplate();
    final int numWorkers = Integer.valueOf(args[1]);
    final int windowSize = Integer.valueOf(args[2]);
    final StateUpdater stateUpdater = new CobwebClusterUpdater(template);
    final StateFactory stateFactory = new MlStormClustererFactory.CobwebClustererFactory(numWorkers, windowSize);
    final QueryFunction<CobwebClustererState, String> queryFunction = new MlStormClustererQuery.CobwebClustererQuery();
    final MlStormSpout features = new MddbFeatureExtractorSpout(args[0], template);
    final StormTopology stormTopology = WekaBaseLearningTopology.buildTopology(features, template, numWorkers, stateUpdater, stateFactory, queryFunction, null, TOPOLOGY_DRPC_NAME, null);

    if (numWorkers == 1) {
        LocalCluster cluster = new LocalCluster();
        cluster.submitTopology(TOPOLOGY_DRPC_NAME, MlStormConfig.getDefaultMlStormConfig(numWorkers), stormTopology);
    } else {
        StormSubmitter.submitTopology(TOPOLOGY_DRPC_NAME, MlStormConfig.getDefaultMlStormConfig(numWorkers), stormTopology);
    }
}
 

import storm.trident.state.StateFactory; //导入依赖的package包/类
public static void main(String[] args) throws AlreadyAliveException, InvalidTopologyException {
    if (args.length < 5) {
        System.err.println(" Where are all the arguments? -- use args -- folder numWorkers windowSize k parallelism");
        return;
    }

    final FieldTemplate template = new MlStormFieldTemplate();
    final int numWorkers = Integer.valueOf(args[1]);
    final int windowSize = Integer.valueOf(args[2]);
    final int k = Integer.valueOf(args[3]);
    final int parallelism = Integer.valueOf(args[4]);
    final StateUpdater stateUpdater = new KmeansClusterUpdater(template);
    final StateFactory stateFactory = new MlStormClustererFactory.KmeansClustererFactory(k, windowSize, template);
    final QueryFunction<KmeansClustererState, String> queryFunction = new MlStormClustererQuery.KmeansClustererQuery();
    final QueryFunction<KmeansClustererState, String> parameterUpdateFunction = new MlStormClustererQuery.KmeansNumClustersUpdateQuery();
    final MlStormSpout features = new MddbFeatureExtractorSpout(args[0], template);
    final StormTopology stormTopology = buildTopology(features, template, parallelism, stateUpdater, stateFactory, queryFunction, parameterUpdateFunction, "kmeans", "kUpdate");

    if (numWorkers == 1) {
        LocalCluster cluster = new LocalCluster();
        cluster.submitTopology("kmeans", MlStormConfig.getDefaultMlStormConfig(numWorkers), stormTopology);
    } else {
        StormSubmitter.submitTopology("kmeans", MlStormConfig.getDefaultMlStormConfig(numWorkers), stormTopology);
    }
}
 

import storm.trident.state.StateFactory; //导入依赖的package包/类
public static void main(String[] args) throws Exception {
    if (args.length < 4) {
        Logger.getAnonymousLogger().log(Level.SEVERE, " Where are all the arguments? -- use args -- file numWorkers windowSize parallelism");
        System.exit(-1);
    }

    final FieldTemplate template = new MlStormFieldTemplate();
    final int numWorkers = Integer.valueOf(args[1]);
    final int windowSize = Integer.valueOf(args[2]);
    final int parallelism = Integer.valueOf(args[3]);
    final StateUpdater stateUpdater = new BinaryClassifierStateUpdater(template);
    final StateFactory stateFactory = new BinaryClassifierFactory(WekaClassificationAlgorithms.svm.name(), windowSize, template, null /* weka.core.Utils.splitOptions("-C 1.0 -L 0.0010 -P 1.0E-12 -N 0 -V -1 -W 1 -K \"weka.classifiers.functions.supportVector.PolyKernel -C 250007 -E 1.0\"")*/);
    final QueryFunction<MlStormWekaState, Integer> queryFunction = new BinaryClassifierQuery.SvmQuery();
    final QueryFunction<KmeansClustererState, String> parameterUpdateFunction = null;
    final MlStormSpout features = new AustralianElectricityPricingSpout(args[0], template);
    final StormTopology stormTopology = WekaBaseLearningTopology.buildTopology(features, template, parallelism, stateUpdater, stateFactory, queryFunction, parameterUpdateFunction, "svm", "svmUpdate");

    if (numWorkers == 1) {
        LocalCluster cluster = new LocalCluster();
        cluster.submitTopology("SVM", MlStormConfig.getDefaultMlStormConfig(numWorkers), stormTopology);
    } else {
        StormSubmitter.submitTopology("SVM", MlStormConfig.getDefaultMlStormConfig(numWorkers), stormTopology);
    }
}
 

import storm.trident.state.StateFactory; //导入依赖的package包/类
private static StormTopology buildTopology(final MlStormSpout mlStormSpout,
                                           final int parallelism,
                                           final int pcaRowWidth,
                                           final int numPrincipalComponents,
                                           final FieldTemplate template) {
    final TridentTopology topology = new TridentTopology();
    final Stream sensorStream = topology.newStream(FieldTemplate.FieldConstants.PCA.PCA, mlStormSpout);
    final StateFactory pcaFactory = new WindowedPcaFactory(pcaRowWidth, numPrincipalComponents, template);

    final TridentState principalComponents =
            sensorStream
                    .partitionPersist(pcaFactory, new Fields(template.getKeyField(), template.getFeatureVectorField()), new PrincipalComponentUpdater(template))
                    .parallelismHint(parallelism);


    topology.newDRPCStream(FieldTemplate.FieldConstants.PCA.PCA_DRPC)
            .broadcast()
            .stateQuery(principalComponents, new Fields(FieldTemplate.FieldConstants.ARGS), new PrincipalComponentsQuery(), new Fields(FieldTemplate.FieldConstants.PCA.PCA_COMPONENTS))
            .project(new Fields(FieldTemplate.FieldConstants.PCA.PCA_COMPONENTS))
            .aggregate(new Fields(FieldTemplate.FieldConstants.PCA.PCA_COMPONENTS), new PrincipalComponentsAggregator(), new Fields(FieldTemplate.FieldConstants.PCA.PCA_EIGEN))
            .project(new Fields(FieldTemplate.FieldConstants.PCA.PCA_EIGEN));

    return topology.build();
}
 

import storm.trident.state.StateFactory; //导入依赖的package包/类
public static void main(String[] args) throws Exception {
//        StateFactory stateFactory = MongoState.nonTransactional("mongodb://127.0.0.1/test.words", Word.class);
//        StateFactory stateFactory = MongoState.opaque("mongodb://127.0.0.1/test.words", Word.class);
        StateFactory stateFactory = MongoState.transactional("mongodb://127.0.0.1/test.words", Word.class);


        Config conf = new Config();
        conf.setMaxSpoutPending(20);
        if (args.length == 0) {
            LocalDRPC drpc = new LocalDRPC();
            LocalCluster cluster = new LocalCluster();
            cluster.submitTopology("wordCounter", conf, buildTopology(drpc, stateFactory));
            for (int i = 0; i < 100; i++) {
                long startDate = System.nanoTime();
                String result = drpc.execute("words", "cat the dog jumped");
                long endDate = System.nanoTime() - startDate;
                System.out.println("DRPC RESULT: " + result + " took: " + endDate / 1000000);
                Thread.sleep(100);
            }
            cluster.shutdown();
        } else {
            conf.setNumWorkers(3);
            StormSubmitter.submitTopology(args[0], conf, buildTopology(null, stateFactory));
        }
    }
 

import storm.trident.state.StateFactory; //导入依赖的package包/类
public static StormTopology buildTopology(Config conf, LocalDRPC drpc) {

        TridentTopology topology = new TridentTopology();

        //Kafka Spout
        BrokerHosts zk = new ZkHosts(conf.get(CrawlerConfig.KAFKA_CONSUMER_HOST_NAME) + ":" +conf.get(CrawlerConfig.KAFKA_CONSUMER_HOST_PORT));
        TridentKafkaConfig kafkaConfig = new TridentKafkaConfig(zk, (String) conf.get(CrawlerConfig.KAFKA_TOPIC_DOCUMENT_NAME));
        kafkaConfig.scheme = new SchemeAsMultiScheme(new StringScheme());
        OpaqueTridentKafkaSpout spout = new OpaqueTridentKafkaSpout(kafkaConfig);

        //ElasticSearch Persistent State
        Settings esSettings = ImmutableSettings.settingsBuilder()
                .put("storm.elasticsearch.cluster.name", conf.get(CrawlerConfig.ELASTICSEARCH_CLUSTER_NAME))
                .put("storm.elasticsearch.hosts", conf.get(CrawlerConfig.ELASTICSEARCH_HOST_NAME) + ":" + conf.get(CrawlerConfig.ELASTICSEARCH_HOST_PORT))
                .build();
        StateFactory esStateFactory = new ESIndexState.Factory<JSONObject>(new ClientFactory.NodeClient(esSettings.getAsMap()), JSONObject.class);
        TridentState esStaticState = topology.newStaticState(esStateFactory);

        String esIndex = (String)(conf.get(CrawlerConfig.ELASTICSEARCH_INDEX_NAME));
        topology.newStream("docstream",spout)
                .each( new Fields("str"), new SplitDocStreamArgs(), new Fields("filename", "task", "user", "content"))
                .each( new Fields("filename", "task", "user"), new PrintFilter("Kafka"))
                .each( new Fields("filename","task","user","content"), new PrepareDocForElasticSearch(), new Fields("index","type","id","source") )
                .partitionPersist(esStateFactory, new Fields("index","type","id","source"), new ESIndexUpdater<String>(new ESTridentTupleMapper()), new Fields());

        return topology.build();
    }
 

import storm.trident.state.StateFactory; //导入依赖的package包/类
public StormTopology buildTopology() {
    TridentTopology topology = new TridentTopology();

    SamevalGenerator dataGen = new SamevalGenerator();

    StateFactory mapState = new MemoryMapState.Factory();

    TridentState counterState = topology.newStream("CounterGen", dataGen)
            .groupBy(new Fields(Names.TIME_STAMP_FLD))
            .persistentAggregate(mapState, new Fields(Names.USER_ID_FLD),
                    new HLLAggregator(Names.USER_ID_FLD),
                    new Fields("ItemCounter"));

    topology.newDRPCStream("CountItemStream", localDRPC)

            .each(new Fields("args"), new Split(), new Fields("FLD"))
            .each(new Fields("FLD"), new DataTypeConvert(new Integer(1)), new Fields(Names.MIN_OF_DAY_FLD))
            .each(new Fields(Names.MIN_OF_DAY_FLD), new Debug())
            .stateQuery(counterState, new Fields(Names.MIN_OF_DAY_FLD), new MapGet(), new Fields(Names.COUNTER_VALS_FLD))
            .each(new Fields(Names.COUNTER_VALS_FLD), new FilterNull())

                    //.each(new Fields("CounterVals"), new HLLToStrConverter("CounterVals"), new Fields("UniqueItems"));
            .each(new Fields(Names.COUNTER_VALS_FLD), new HLLToStrConverter(Names.COUNTER_VALS_FLD), new Fields("UniqueItems"))
            .project(new Fields("UniqueItems"));

    return topology.build();
}
 

import storm.trident.state.StateFactory; //导入依赖的package包/类
public static StormTopology buildTopology(String hdfsUrl){
    FixedBatchSpout spout = new FixedBatchSpout(new Fields("sentence", "key"), 1000, new Values("the cow jumped over the moon", 1l),
            new Values("the man went to the store and bought some candy", 2l), new Values("four score and seven years ago", 3l),
            new Values("how many apples can you eat", 4l), new Values("to be or not to be the person", 5l));
    spout.setCycle(true);

    TridentTopology topology = new TridentTopology();
    Stream stream = topology.newStream("spout1", spout);

    Fields hdfsFields = new Fields("sentence", "key");

    FileNameFormat fileNameFormat = new DefaultFileNameFormat()
            .withPath("/trident")
            .withPrefix("trident")
            .withExtension(".txt");

    RecordFormat recordFormat = new DelimitedRecordFormat()
            .withFields(hdfsFields);

    FileRotationPolicy rotationPolicy = new FileSizeRotationPolicy(5.0f, FileSizeRotationPolicy.Units.MB);

    HdfsState.Options options = new HdfsState.HdfsFileOptions()
            .withFileNameFormat(fileNameFormat)
            .withRecordFormat(recordFormat)
            .withRotationPolicy(rotationPolicy)
            .withFsUrl(hdfsUrl);

    StateFactory factory = new HdfsStateFactory().withOptions(options);

    TridentState state = stream
            .partitionPersist(factory, hdfsFields, new HdfsUpdater(), new Fields());

    return topology.build();
}
 

import storm.trident.state.StateFactory; //导入依赖的package包/类
public static StormTopology buildTopology(String hdfsUrl){
    FixedBatchSpout spout = new FixedBatchSpout(new Fields("sentence", "key"), 1000, new Values("the cow jumped over the moon", 1l),
            new Values("the man went to the store and bought some candy", 2l), new Values("four score and seven years ago", 3l),
            new Values("how many apples can you eat", 4l), new Values("to be or not to be the person", 5l));
    spout.setCycle(true);

    TridentTopology topology = new TridentTopology();
    Stream stream = topology.newStream("spout1", spout);

    Fields hdfsFields = new Fields("sentence", "key");

    FileNameFormat fileNameFormat = new DefaultFileNameFormat()
            .withPath("/trident")
            .withPrefix("trident")
            .withExtension(".seq");

    FileRotationPolicy rotationPolicy = new FileSizeRotationPolicy(5.0f, FileSizeRotationPolicy.Units.MB);

    HdfsState.Options seqOpts = new HdfsState.SequenceFileOptions()
            .withFileNameFormat(fileNameFormat)
            .withSequenceFormat(new DefaultSequenceFormat("key", "sentence"))
            .withRotationPolicy(rotationPolicy)
            .withFsUrl(hdfsUrl)
            .addRotationAction(new MoveFileAction().toDestination("/dest2/"));

    StateFactory factory = new HdfsStateFactory().withOptions(seqOpts);

    TridentState state = stream
            .partitionPersist(factory, hdfsFields, new HdfsUpdater(), new Fields());

    return topology.build();
}
 

import storm.trident.state.StateFactory; //导入依赖的package包/类
private static void assertArguments(IRichSpout spout, int parallelism, List<StateUpdater> stateUpdaters, List<StateFactory> stateFactories, List<QueryFunction> queryFunctions, List<String> drpcQueryFunctionNames, ReducerAggregator drpcPartitionResultAggregator, StateFactory metaStateFactory, StateUpdater metaStateUpdater, QueryFunction metaQueryFunction) {
    assert spout != null;
    assert parallelism != 0;
    assert stateFactories != null;
    assert queryFunctions != null;
    assert drpcPartitionResultAggregator != null;
    assert stateUpdaters != null;
    assert drpcQueryFunctionNames != null;
    assert metaQueryFunction != null;
    assert metaStateFactory != null;
    assert metaStateUpdater != null;
}
 

import storm.trident.state.StateFactory; //导入依赖的package包/类
public static StormTopology buildTopology(LocalDRPC drpc, StateFactory stateFactory) {
    FixedBatchSpout spout = new FixedBatchSpout(new Fields("sentence"), 3,
            new Values("the cow jumped over the moon"),
            new Values("the man went to the store and bought some candy"),
            new Values("four score and seven years ago"),
            new Values("how many apples can you eat"),
            new Values("to be or not to be the person"));
    spout.setCycle(true);

    TridentTopology topology = new TridentTopology();
    topology.build();
    TridentState wordCounts =
            topology.newStream("spout1", spout)
                    .parallelismHint(16)
                    .each(new Fields("sentence"), new Split(), new Fields("word"))
                    .groupBy(new Fields("word"))
                    .persistentAggregate(stateFactory, new Fields("word"), new WordCount(), new Fields("count"))
                    .parallelismHint(16);

    topology.newDRPCStream("words", drpc)
            .each(new Fields("args"), new Split(), new Fields("word"))
            .groupBy(new Fields("word"))
            .stateQuery(wordCounts, new Fields("word"), new MapGet(), new Fields("count"))
            .each(new Fields("count"), new FilterNull())
            .aggregate(new Fields("count"), new SumWord(), new Fields("sum"))
    ;

    return topology.build();
}