Java Config.putAll方法代碼示例

import backtype.storm.Config; //導入方法依賴的package包/類
public static void main(String[] args) throws Exception {

        if(args.length < 1){
            System.err.println("[ERROR] Configuration File Required");
        }
        Config conf = new Config();

        // Store all the configuration in the Storm conf object
        conf.putAll(readConfigFile(args[0]));

        //Second arg should be local in order to run locally
        if(args.length  < 2 || (args.length  == 2 && !args[1].equals("local"))) {
            StormSubmitter.submitTopologyWithProgressBar("crawler_topology", conf, buildTopology(conf, null));
        }
        else {
            LocalDRPC drpc = new LocalDRPC();
            LocalCluster localcluster = new LocalCluster();
            localcluster.submitTopology("crawler_topology",conf,buildTopology(conf, drpc));

            String searchQuery = "elasticsearch";
            System.out.println("---* Result (search): " + drpc.execute("search",  searchQuery));
        }
    }
 

import backtype.storm.Config; //導入方法依賴的package包/類
public static Config loadConf(String resource) throws FileNotFoundException {
    Config conf = new Config();
    Yaml yaml = new Yaml();
    Map ret = (Map) yaml.load(new InputStreamReader(new FileInputStream(
            resource), Charset.defaultCharset()));
    if (ret == null) {
        ret = new HashMap();
    }
    conf.putAll(ret);
    return conf;
}
 

import backtype.storm.Config; //導入方法依賴的package包/類
public static void main(String[] args) throws Exception {
	
    LinearDRPCTopologyBuilder builder = construct();
  
    Config conf = new Config();
    conf.setNumWorkers(6);
    if (args.length != 0) {
    	
    	try {
     	Map yamlConf = LoadConf.LoadYaml(args[0]);
     	if (yamlConf != null) {
     		conf.putAll(yamlConf);
     	}
    	}catch (Exception e) {
    		System.out.println("Input " + args[0] + " isn't one yaml ");
    	}
    	
    	
    	StormSubmitter.submitTopology(TOPOLOGY_NAME, conf, builder.createRemoteTopology());
    }else {
    
    
        conf.setMaxTaskParallelism(3);
        LocalDRPC drpc = new LocalDRPC();
        LocalCluster cluster = new LocalCluster();
        cluster.submitTopology(TOPOLOGY_NAME, conf, builder.createLocalTopology(drpc));
        
        JStormUtils.sleepMs(50000);
        
        String[] urlsToTry = new String[] { "foo.com/blog/1", "engineering.twitter.com/blog/5", "notaurl.com"};
        for(String url: urlsToTry) {
            System.out.println("Reach of " + url + ": " + drpc.execute(TOPOLOGY_NAME, url));
        }
        
        cluster.shutdown();
        drpc.shutdown();
    }
}
 

import backtype.storm.Config; //導入方法依賴的package包/類
public static void main(String[] args) throws Exception {
  MemoryTransactionalSpout spout = new MemoryTransactionalSpout(DATA, new Fields("word"), PARTITION_TAKE_PER_BATCH);
  TransactionalTopologyBuilder builder = new TransactionalTopologyBuilder("global-count", "spout", spout, 3);
  builder.setBolt("partial-count", new BatchCount(), 5).noneGrouping("spout");
  builder.setBolt("sum", new UpdateGlobalCount()).globalGrouping("partial-count");

  Config config = new Config();
  config.setDebug(true);
  config.setMaxSpoutPending(3);
  
  if (args.length == 0){
      LocalCluster cluster = new LocalCluster();    
  
      cluster.submitTopology("global-count-topology", config, builder.buildTopology());
  
      Thread.sleep(100000);
      cluster.shutdown();
  }else {
  	
      config.setNumWorkers(3);
  	try {
      	Map yamlConf = LoadConf.LoadYaml(args[0]);
      	if (yamlConf != null) {
      		config.putAll(yamlConf);
      	}
  	}catch (Exception e) {
  		System.out.println("Input " + args[0] + " isn't one yaml ");
  	}

      StormSubmitter.submitTopology("global", config, builder.buildTopology());
  }
}
 

import backtype.storm.Config; //導入方法依賴的package包/類
public static void main(String[] args) throws Exception {

        if(args.length != 2){
            System.err.println("[ERROR] Configuration File Required");
        }
        Config conf = new Config();

        //Map topologyConfig = readConfigFile(args[0]);
        //conf.putAll(topologyConfig);

        // Store all the configuration in the Storm conf object
        conf.putAll(ConfigReader.readConfigFile(args[0]));

        //Second arg should be local in order to run locally
        if(args[1].equals("local"))
        {
            LocalDRPC drpc = new LocalDRPC();
            LocalCluster localcluster = new LocalCluster();
            localcluster.submitTopology("url_event_processing",conf,buildTopology(conf, drpc));

            String searchQuery = "HoloLens crawl_test";
            System.out.println("---* Result: " + drpc.execute("search",  searchQuery));
        }
        else
        {
            StormSubmitter.submitTopologyWithProgressBar("url_event_processing", conf, buildTopology(conf, null));
        }
    }
 

import backtype.storm.Config; //導入方法依賴的package包/類
public static void main(String[] args) throws Exception{
    if(args.length != 2){
        System.err.println("[ERROR] Configuration File Required");
    }
    Config conf = new Config();

    //Map topologyConfig = readConfigFile(args[0]);
    //conf.putAll(topologyConfig);

    // Store all the configuration in the Storm conf object
    conf.putAll(ConfigReader.readConfigFile(args[0]));

    //Second arg should be local in order to run locally
    if(args[1].equals("local"))
    {
        LocalDRPC drpc = new LocalDRPC();
        LocalCluster localcluster = new LocalCluster();
        localcluster.submitTopology("doc_event_processing",conf,buildTopology(conf, drpc));

        String searchQuery = "HoloLens crawl_test";
        System.out.println("---* Result: " + drpc.execute("search",  searchQuery));
    }
    else
    {
        StormSubmitter.submitTopologyWithProgressBar("doc_event_processing", conf, buildTopology(conf, null));
    }
}
 

import backtype.storm.Config; //導入方法依賴的package包/類
@SuppressWarnings({ "rawtypes", "unchecked" })
@Override
public void prepare(Map stormConf, TopologyContext context,
        OutputCollector collector) {

    _collector = collector;

    Config conf = new Config();
    conf.putAll(stormConf);

    checkConfiguration(conf);

    SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss",
            Locale.ENGLISH);
    long start = System.currentTimeMillis();
    LOG.info("[Fetcher #{}] : starting at {}", taskIndex, sdf.format(start));

    // Register a "MultiCountMetric" to count different events in this bolt
    // Storm will emit the counts every n seconds to a special bolt via a
    // system stream
    // The data can be accessed by registering a "MetricConsumer" in the
    // topology
    this.eventCounter = context.registerMetric("fetcher_counter",
            new MultiCountMetric(), 10);

    this.metricGauge = context.registerMetric("fetcher",
            new MultiCountMetric(), 10);

    this.averagedMetrics = context.registerMetric("fetcher_average",
            new MultiReducedMetric(new MeanReducer()), 10);

    protocolFactory = new ProtocolFactory(conf);

    this.fetchQueues = new FetchItemQueues(conf);

    this.taskIndex = context.getThisTaskIndex();

    int threadCount = ConfUtils.getInt(conf, "fetcher.threads.number", 10);
    for (int i = 0; i < threadCount; i++) { // spawn threads
        new FetcherThread(conf).start();
    }

    String urlconfigfile = ConfUtils.getString(conf,
            "urlfilters.config.file", "urlfilters.json");

    if (urlconfigfile != null)
        try {
            urlFilters = new URLFilters(conf, urlconfigfile);
        } catch (IOException e) {
            LOG.error("Exception caught while loading the URLFilters");
            throw new RuntimeException(
                    "Exception caught while loading the URLFilters", e);
        }

    allowRedirs = ConfUtils.getBoolean(stormConf,
            com.digitalpebble.storm.crawler.Constants.AllowRedirParamName,
            true);

    metadataTransfer = MetadataTransfer.getInstance(stormConf);
}
 

import backtype.storm.Config; //導入方法依賴的package包/類
/**
 * The main method.
 * 
 * @param args
 *          the arguments
 */
public static void main(String[] args) {

  List<String> tmpArgs = new ArrayList<String>(Arrays.asList(args));

  boolean isLocal = isLocal(tmpArgs);
  int numWorker = StormSamoaUtils.numWorkers(tmpArgs);

  args = tmpArgs.toArray(new String[0]);

  // convert the arguments into Storm topology
  StormTopology stormTopo = StormSamoaUtils.argsToTopology(args);
  String topologyName = stormTopo.getTopologyName();

  Config conf = new Config();
  conf.putAll(Utils.readStormConfig());
  conf.setDebug(false);

  if (isLocal) {
    // local mode
    conf.setMaxTaskParallelism(numWorker);

    backtype.storm.LocalCluster cluster = new backtype.storm.LocalCluster();
    cluster.submitTopology(topologyName, conf, stormTopo.getStormBuilder().createTopology());

    backtype.storm.utils.Utils.sleep(600 * 1000);

    cluster.killTopology(topologyName);
    cluster.shutdown();

  } else {
    // cluster mode
    conf.setNumWorkers(numWorker);
    try {
      backtype.storm.StormSubmitter.submitTopology(topologyName, conf,
          stormTopo.getStormBuilder().createTopology());
    } catch (backtype.storm.generated.AlreadyAliveException ale) {
      ale.printStackTrace();
    } catch (backtype.storm.generated.InvalidTopologyException ite) {
      ite.printStackTrace();
    }
  }
}
 

import backtype.storm.Config; //導入方法依賴的package包/類
public static void main(String[] args) throws IOException {
  Properties props = StormSamoaUtils.getProperties();

  String uploadedJarLocation = props.getProperty(StormJarSubmitter.UPLOADED_JAR_LOCATION_KEY);
  if (uploadedJarLocation == null) {
    logger.error("Invalid properties file. It must have key {}",
        StormJarSubmitter.UPLOADED_JAR_LOCATION_KEY);
    return;
  }

  List<String> tmpArgs = new ArrayList<String>(Arrays.asList(args));
  int numWorkers = StormSamoaUtils.numWorkers(tmpArgs);

  args = tmpArgs.toArray(new String[0]);
  StormTopology stormTopo = StormSamoaUtils.argsToTopology(args);

  Config conf = new Config();
  conf.putAll(Utils.readStormConfig());
  conf.putAll(Utils.readCommandLineOpts());
  conf.setDebug(false);
  conf.setNumWorkers(numWorkers);

  String profilerOption =
      props.getProperty(StormTopologySubmitter.YJP_OPTIONS_KEY);
  if (profilerOption != null) {
    String topoWorkerChildOpts = (String) conf.get(Config.TOPOLOGY_WORKER_CHILDOPTS);
    StringBuilder optionBuilder = new StringBuilder();
    if (topoWorkerChildOpts != null) {
      optionBuilder.append(topoWorkerChildOpts);
      optionBuilder.append(' ');
    }
    optionBuilder.append(profilerOption);
    conf.put(Config.TOPOLOGY_WORKER_CHILDOPTS, optionBuilder.toString());
  }

  Map<String, Object> myConfigMap = new HashMap<String, Object>(conf);
  StringWriter out = new StringWriter();

  try {
    JSONValue.writeJSONString(myConfigMap, out);
  } catch (IOException e) {
    System.out.println("Error in writing JSONString");
    e.printStackTrace();
    return;
  }

  Config config = new Config();
  config.putAll(Utils.readStormConfig());

  NimbusClient nc = NimbusClient.getConfiguredClient(config);
  String topologyName = stormTopo.getTopologyName();
  try {
    System.out.println("Submitting topology with name: "
        + topologyName);
    nc.getClient().submitTopology(topologyName, uploadedJarLocation,
        out.toString(), stormTopo.getStormBuilder().createTopology());
    System.out.println(topologyName + " is successfully submitted");

  } catch (AlreadyAliveException aae) {
    System.out.println("Fail to submit " + topologyName
        + "\nError message: " + aae.get_msg());
  } catch (InvalidTopologyException ite) {
    System.out.println("Invalid topology for " + topologyName);
    ite.printStackTrace();
  } catch (TException te) {
    System.out.println("Texception for " + topologyName);
    te.printStackTrace();
  }
}
 

import backtype.storm.Config; //導入方法依賴的package包/類
public static Config getConfig(String env, File config) throws IOException {
  ConfigObject configObject = new ConfigSlurper(env).parse(readGroovyConfigScript(config));
  Config stormConf = new Config();
  Map flatten = configObject.flatten();
  stormConf.putAll(flatten);

  Map<String, Class> dataTypes = new HashMap<String, Class>();
  dataTypes.put("topology.workers", Integer.class);
  dataTypes.put("topology.acker.executors", Integer.class);
  dataTypes.put("topology.message.timeout.secs", Integer.class);
  dataTypes.put("topology.max.task.parallelism", Integer.class);
  dataTypes.put("topology.stats.sample.rate", Double.class);

  // this will convert built in properties as storm uses old school properties
  for (Field field : stormConf.getClass().getFields()) {
    if (Modifier.isStatic(field.getModifiers())
      && Modifier.isPublic(field.getModifiers())) {
      String property = field.getName().toLowerCase().replace('_', '.');
      if (property.startsWith("java.")) {
        // don't mess with Java system properties here
        continue;
      }

      Object override = flatten.get(property);
      if (override != null) {
        stormConf.put(property, override);
        System.out.println("Overrode property '" + property + "' with value [" + override + "] from Config.groovy of type " + override.getClass().getName());
      }
      String system = System.getProperty(property, null);
      if (system != null) {
        if (dataTypes.containsKey(property)) {
          Class aClass = dataTypes.get(property);
          try {
            Method valueOf = aClass.getMethod("valueOf", String.class);
            stormConf.put(property, valueOf.invoke(aClass, system));
            System.out.println("Overrode property '" + property + "' with value [" + stormConf.get(property) + "] from -D System property of type " + aClass.getName());
          } catch (Exception e) {
            throw new RuntimeException(e.getMessage(), e);
          }
        } else {
          stormConf.put(property, system);
          System.out.println("Overrode property '" + property + "' with String value [" + system + "] from -D System property");
        }
      }
    }
  }

  return stormConf;
}