Java StreamsConfigurator类代码示例

import org.apache.streams.config.StreamsConfigurator; //导入依赖的package包/类
public static void main(String[] args)
{
  LOGGER.info(StreamsConfigurator.getConfig().toString());

  TwitterUserstreamElasticsearch userstream = new TwitterUserstreamElasticsearch();
  new Thread(userstream).start();

}
 

import org.apache.streams.config.StreamsConfigurator; //导入依赖的package包/类
@Override
public void run() {

  TwitterStreamConfiguration twitterStreamConfiguration = config.getTwitter();
  ElasticsearchWriterConfiguration elasticsearchWriterConfiguration = config.getElasticsearch();

  TwitterStreamProvider stream = new TwitterStreamProvider(twitterStreamConfiguration);
  ActivityConverterProcessor converter = new ActivityConverterProcessor();
  VerbDefinitionDropFilter noDeletesProcessor = new VerbDefinitionDropFilter(Stream.of(deleteVerbDefinition).collect(Collectors.toSet()));
  ElasticsearchPersistWriter writer = new ElasticsearchPersistWriter(elasticsearchWriterConfiguration);
  VerbDefinitionKeepFilter deleteOnlyProcessor = new VerbDefinitionKeepFilter(Stream.of(deleteVerbDefinition).collect(Collectors.toSet()));
  SetDeleteIdProcessor setDeleteIdProcessor = new SetDeleteIdProcessor();
  ElasticsearchPersistDeleter deleter = new ElasticsearchPersistDeleter(elasticsearchWriterConfiguration);

  LocalRuntimeConfiguration localRuntimeConfiguration =
      StreamsJacksonMapper.getInstance().convertValue(StreamsConfigurator.detectConfiguration(), LocalRuntimeConfiguration.class);
  StreamBuilder builder = new LocalStreamBuilder(localRuntimeConfiguration);

  builder.newPerpetualStream(TwitterStreamProvider.class.getCanonicalName(), stream);
  builder.addStreamsProcessor(ActivityConverterProcessor.class.getCanonicalName(), converter, 2, TwitterStreamProvider.class.getCanonicalName());
  builder.addStreamsProcessor(VerbDefinitionDropFilter.class.getCanonicalName(), noDeletesProcessor, 1, ActivityConverterProcessor.class.getCanonicalName());
  builder.addStreamsPersistWriter(ElasticsearchPersistWriter.class.getCanonicalName(), writer, 1, VerbDefinitionDropFilter.class.getCanonicalName());
  builder.addStreamsProcessor(VerbDefinitionKeepFilter.class.getCanonicalName(), deleteOnlyProcessor, 1, ActivityConverterProcessor.class.getCanonicalName());
  builder.addStreamsProcessor(SetDeleteIdProcessor.class.getCanonicalName(), setDeleteIdProcessor, 1, VerbDefinitionKeepFilter.class.getCanonicalName());
  builder.addStreamsPersistWriter(ElasticsearchPersistDeleter.class.getCanonicalName(), deleter, 1, SetDeleteIdProcessor.class.getCanonicalName());

  builder.start();

}
 

import org.apache.streams.config.StreamsConfigurator; //导入依赖的package包/类
@Override
public void run() {

  MongoPersistReader mongoPersistReader = new MongoPersistReader(config.getSource());

  ElasticsearchPersistWriter elasticsearchPersistWriter = new ElasticsearchPersistWriter(config.getDestination());

  LocalRuntimeConfiguration localRuntimeConfiguration =
      StreamsJacksonMapper.getInstance().convertValue(StreamsConfigurator.detectConfiguration(), LocalRuntimeConfiguration.class);
  StreamBuilder builder = new LocalStreamBuilder(localRuntimeConfiguration);

  builder.newPerpetualStream(MongoPersistReader.class.getCanonicalName(), mongoPersistReader);
  builder.addStreamsPersistWriter(ElasticsearchPersistWriter.class.getCanonicalName(), elasticsearchPersistWriter, 1, MongoPersistReader.class.getCanonicalName());
  builder.start();
}
 

import org.apache.streams.config.StreamsConfigurator; //导入依赖的package包/类
@Override
public void run() {

  ElasticsearchPersistReader elasticsearchPersistReader = new ElasticsearchPersistReader(config.getSource());

  ElasticsearchPersistWriter elasticsearchPersistWriter = new ElasticsearchPersistWriter(config.getDestination());

  LocalRuntimeConfiguration localRuntimeConfiguration =
      StreamsJacksonMapper.getInstance().convertValue(StreamsConfigurator.detectConfiguration(), LocalRuntimeConfiguration.class);
  StreamBuilder builder = new LocalStreamBuilder(localRuntimeConfiguration);

  builder.newPerpetualStream(ElasticsearchPersistReader.class.getCanonicalName(), elasticsearchPersistReader);
  builder.addStreamsPersistWriter(ElasticsearchPersistWriter.class.getCanonicalName(), elasticsearchPersistWriter, 1, ElasticsearchPersistReader.class.getCanonicalName());
  builder.start();
}
 

import org.apache.streams.config.StreamsConfigurator; //导入依赖的package包/类
@Override
public void run() {

  WebHdfsPersistReader webHdfsPersistReader = new WebHdfsPersistReader(config.getSource());
  ElasticsearchPersistWriter elasticsearchPersistWriter = new ElasticsearchPersistWriter(config.getDestination());

  LocalRuntimeConfiguration localRuntimeConfiguration =
      StreamsJacksonMapper.getInstance().convertValue(StreamsConfigurator.detectConfiguration(), LocalRuntimeConfiguration.class);
  StreamBuilder builder = new LocalStreamBuilder(localRuntimeConfiguration);

  builder.newPerpetualStream(WebHdfsPersistReader.class.getCanonicalName(), webHdfsPersistReader);
  builder.addStreamsPersistWriter(ElasticsearchPersistWriter.class.getCanonicalName(), elasticsearchPersistWriter, 1, WebHdfsPersistReader.class.getCanonicalName());
  builder.start();
}
 

import org.apache.streams.config.StreamsConfigurator; //导入依赖的package包/类
@Override
public void run() {

  ElasticsearchPersistReader elasticsearchPersistReader = new ElasticsearchPersistReader(config.getSource());
  WebHdfsPersistWriter hdfsPersistWriter = new WebHdfsPersistWriter(config.getDestination());

  LocalRuntimeConfiguration localRuntimeConfiguration =
      StreamsJacksonMapper.getInstance().convertValue(StreamsConfigurator.detectConfiguration(), LocalRuntimeConfiguration.class);
  StreamBuilder builder = new LocalStreamBuilder(localRuntimeConfiguration);

  builder.newPerpetualStream(ElasticsearchPersistReader.class.getCanonicalName(), elasticsearchPersistReader);
  builder.addStreamsPersistWriter(WebHdfsPersistWriter.class.getCanonicalName(), hdfsPersistWriter, 1, ElasticsearchPersistReader.class.getCanonicalName());
  builder.start();
}
 

import org.apache.streams.config.StreamsConfigurator; //导入依赖的package包/类
public void run() {

    TwitterFollowingConfiguration twitterFollowingConfiguration = config.getTwitter();
    TwitterFollowingProvider followingProvider = new TwitterFollowingProvider(twitterFollowingConfiguration);
    TypeConverterProcessor converter = new TypeConverterProcessor(String.class);

    List<DocumentClassifier> classifiers = Stream.of((DocumentClassifier) new TwitterDocumentClassifier()).collect(Collectors.toList());
    List<ActivityConverter> converters = Stream.of((ActivityConverter) new TwitterFollowActivityConverter()).collect(Collectors.toList());
    ActivityConverterProcessorConfiguration activityConverterProcessorConfiguration =
        new ActivityConverterProcessorConfiguration()
            .withClassifiers(classifiers)
            .withConverters(converters);
    ActivityConverterProcessor activity = new ActivityConverterProcessor(activityConverterProcessorConfiguration);

    Neo4jConfiguration neo4jConfiguration = config.getNeo4j();
    Neo4jBoltPersistWriter graphPersistWriter = new Neo4jBoltPersistWriter(neo4jConfiguration);
    graphPersistWriter.prepare(neo4jConfiguration);

    LocalRuntimeConfiguration localRuntimeConfiguration =
        StreamsJacksonMapper.getInstance().convertValue(StreamsConfigurator.detectConfiguration(), LocalRuntimeConfiguration.class);
    StreamBuilder builder = new LocalStreamBuilder(localRuntimeConfiguration);

    builder.newPerpetualStream(TwitterFollowingProvider.class.getCanonicalName(), followingProvider);
    builder.addStreamsProcessor(TypeConverterProcessor.class.getCanonicalName(), converter, 1, TwitterFollowingProvider.class.getCanonicalName());
    builder.addStreamsProcessor(ActivityConverterProcessor.class.getCanonicalName(), activity, 1, TypeConverterProcessor.class.getCanonicalName());
    builder.addStreamsPersistWriter(Neo4jBoltPersistWriter.class.getCanonicalName(), graphPersistWriter, 1, ActivityConverterProcessor.class.getCanonicalName());

    builder.start();
  }
 

import org.apache.streams.config.StreamsConfigurator; //导入依赖的package包/类
public void run() {

    TwitterTimelineProvider provider = new TwitterTimelineProvider(config.getTwitter());
    ActivityConverterProcessor converter = new ActivityConverterProcessor();
    ElasticsearchPersistWriter writer = new ElasticsearchPersistWriter(config.getElasticsearch());

    LocalRuntimeConfiguration localRuntimeConfiguration =
        StreamsJacksonMapper.getInstance().convertValue(StreamsConfigurator.detectConfiguration(), LocalRuntimeConfiguration.class);
    StreamBuilder builder = new LocalStreamBuilder(localRuntimeConfiguration);

    builder.newPerpetualStream(TwitterTimelineProvider.class.getCanonicalName(), provider);
    builder.addStreamsProcessor(ActivityConverterProcessor.class.getCanonicalName(), converter, 2, TwitterTimelineProvider.class.getCanonicalName());
    builder.addStreamsPersistWriter(ElasticsearchPersistWriter.class.getCanonicalName(), writer, 1, ActivityConverterProcessor.class.getCanonicalName());
    builder.start();
  }
 

import org.apache.streams.config.StreamsConfigurator; //导入依赖的package包/类
public MongoElasticsearchSync() {
  this(new ComponentConfigurator<MongoElasticsearchSyncConfiguration>(MongoElasticsearchSyncConfiguration.class).detectConfiguration(StreamsConfigurator.getConfig()));
}
 

import org.apache.streams.config.StreamsConfigurator; //导入依赖的package包/类
public HdfsElasticsearch() {
  this(new ComponentConfigurator<>(HdfsElasticsearchConfiguration.class).detectConfiguration(StreamsConfigurator.getConfig()));
}
 

import org.apache.streams.config.StreamsConfigurator; //导入依赖的package包/类
public static void main(String[] args)
{
  LOGGER.info(StreamsConfigurator.getConfig().toString());
  HdfsElasticsearch restore = new HdfsElasticsearch();
  new Thread(restore).start();
}
 

import org.apache.streams.config.StreamsConfigurator; //导入依赖的package包/类
public ElasticsearchHdfs() {
  this(new ComponentConfigurator<>(ElasticsearchHdfsConfiguration.class).detectConfiguration(StreamsConfigurator.getConfig()));
}
 

import org.apache.streams.config.StreamsConfigurator; //导入依赖的package包/类
public static void main(String[] args)
{
  LOGGER.info(StreamsConfigurator.getConfig().toString());
  ElasticsearchHdfs backup = new ElasticsearchHdfs();
  new Thread(backup).start();
}
 

import org.apache.streams.config.StreamsConfigurator; //导入依赖的package包/类
public TwitterFollowNeo4j() {
  this(new ComponentConfigurator<>(TwitterFollowNeo4jConfiguration.class).detectConfiguration(StreamsConfigurator.getConfig()));
}
 

import org.apache.streams.config.StreamsConfigurator; //导入依赖的package包/类
public TwitterHistoryElasticsearch() {
  this(new ComponentConfigurator<>(TwitterHistoryElasticsearchConfiguration.class).detectConfiguration(StreamsConfigurator.getConfig()));
}