Java SpoutOutputCollector类代码示例

import backtype.storm.spout.SpoutOutputCollector; //导入依赖的package包/类
public EmitState next(SpoutOutputCollector collector) {
    if (_waitingToEmit.isEmpty()) {
        fill();
    }
    while (true) {
        MessageAndRealOffset toEmit = _waitingToEmit.pollFirst();
        if (toEmit == null) {
            return EmitState.NO_EMITTED;
        }
        Iterable<List<Object>> tups = KafkaUtils.generateTuples(_spoutConfig, toEmit.msg);
        if (tups != null) {
            for (List<Object> tup : tups) {
                collector.emit(tup, new KafkaMessageId(_partition, toEmit.offset));
            }
            break;
        } else {
            ack(toEmit.offset);
        }
    }
    if (!_waitingToEmit.isEmpty()) {
        return EmitState.EMITTED_MORE_LEFT;
    } else {
        return EmitState.EMITTED_END;
    }
}
 

import backtype.storm.spout.SpoutOutputCollector; //导入依赖的package包/类
public void open(Map conf, TopologyContext ctx, SpoutOutputCollector collector) {
	this.collector = collector;
	this.history = new ProcessedHistory();
	this.subreddit = (String) conf.get("subreddit");
	
	try {
		this.subredditCommentsfeedURL = new URL((String)conf.get("feedURL"));
	} catch (MalformedURLException e) {
		throw new RuntimeException(e);
	}
	LOG.info("Spout subreddit:{} feedURL:{}", this.subreddit, this.subredditCommentsfeedURL);
	
	if (conf.containsKey("sentimentData")) {
		LOG.info("Spouts can also see sentimentData");
	}
}
 

import backtype.storm.spout.SpoutOutputCollector; //导入依赖的package包/类
/**
 * Open file with stock tick data and read into List object.
 */
@Override
public void open(Map map,
                 TopologyContext context,
                 SpoutOutputCollector outputCollector) {
  this.outputCollector = outputCollector;

  try {
    ticks = 
      IOUtils.readLines(ClassLoader.getSystemResourceAsStream(
 "NASDAQ_daily_prices_A.csv"),
        Charset.defaultCharset().name());
  } catch (IOException e) {
      throw new RuntimeException(e);
  }
}
 

import backtype.storm.spout.SpoutOutputCollector; //导入依赖的package包/类
public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
	try {
		_collector = collector;

		ru = new RedisUtil();
		redis = ru.getJedisInstance();

		SchedulerFactory schedulerFactory = new StdSchedulerFactory();
		Scheduler scheduler = schedulerFactory.getScheduler();
		_qManager = new QuartzManager();
		_qManager.setScheduler(scheduler);
		PlatformUtils.initRegisterProject(_qManager);
		scheduler.start();

		//init Hbase tables
		CreateTable.initHbaseTable();
	}catch(Exception ex){
		logger.error("error:"+MySysLogger.formatException(ex));
		ex.printStackTrace();
	}
}
 

import backtype.storm.spout.SpoutOutputCollector; //导入依赖的package包/类
public void open(Map conf, TopologyContext context,
                 SpoutOutputCollector collector) {
    queue = new LinkedBlockingQueue<String>(1000);
    this.collector = collector;

    StatusListener listener = new StatusListener() {
        public void onStatus(Status status) {
            queue.offer(TwitterObjectFactory.getRawJSON(status));
        }

        public void onDeletionNotice(StatusDeletionNotice sdn) { }
        public void onTrackLimitationNotice(int i) { }
        public void onScrubGeo(long l, long l1) { }
        public void onStallWarning(StallWarning stallWarning) { }
        public void onException(Exception e) { }
    };

    ConfigurationBuilder cb = new ConfigurationBuilder();
    cb.setJSONStoreEnabled(true);

    TwitterStreamFactory factory = new TwitterStreamFactory(cb.build());
    twitterStream = factory.getInstance();
    twitterStream.addListener(listener);
    twitterStream.filter(new FilterQuery().language("en").track("trump"));
}
 

import backtype.storm.spout.SpoutOutputCollector; //导入依赖的package包/类
@Override
public void open(Map arg0, TopologyContext arg1, SpoutOutputCollector arg2) {
  /*
   * FileReader fileReader; try { fileReader = new
   * FileReader("C:/proj/Migration/apache-storm-0.9.4/apache-storm-0.9.4/logs/InPut.txt");
   * bufferedReader =new BufferedReader(fileReader); _collector= arg2; } catch
   * (FileNotFoundException e) { // TODO Auto-generated catch block e.printStackTrace(); }
   */

  try {
    _collector = arg2;
    setUpConnection();
  } catch (JMSException e) {
    LOGGER.error(e);
  }

}
 

import backtype.storm.spout.SpoutOutputCollector; //导入依赖的package包/类
@Override
public void open(Map conf, TopologyContext context,
		SpoutOutputCollector collector) {
	batchQueue = new LinkedBlockingQueue<BatchSpoutMsgId>();
	this.collector = collector;
	this.conf = conf;
	taskName = context.getThisComponentId() + "_" + context.getThisTaskId();
	
	intervalCheck = new IntervalCheck();

	try {
		zkClient = BatchCommon.getZkClient(conf);

		initMsgId();

	} catch (Exception e) {
		LOG.error("", e);
		throw new RuntimeException("Failed to init");
	}
	LOG.info("Successfully open " + taskName);
}
 

import backtype.storm.spout.SpoutOutputCollector; //导入依赖的package包/类
public void prepare(TaskSendTargets sendTargets, TaskTransfer transferFn,
		TopologyContext topologyContext) {

	
	// collector, in fact it call send_spout_msg
	this.output_collector = new SpoutCollector(taskId, spout, task_stats,
			sendTargets, storm_conf, transferFn, pending, topologyContext,
			exeQueue, report_error);

	try {
		WorkerClassLoader.switchThreadContext();
		this.spout.open(storm_conf, userTopologyCtx,
				new SpoutOutputCollector(output_collector));
	} catch (Throwable e) {
		error = e;
		LOG.error("spout open error ", e);
		report_error.report(e);
	} finally {
		WorkerClassLoader.restoreThreadContext();
	}

	LOG.info("Successfully create SpoutExecutors " + idStr);

}
 

import backtype.storm.spout.SpoutOutputCollector; //导入依赖的package包/类
@Override
public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
    _throttler = new WindowedTimeThrottler((Number)conf.get(Config.TOPOLOGY_TRIDENT_BATCH_EMIT_INTERVAL_MILLIS), 1);
    for(String spoutId: _managedSpoutIds) {
        _states.add(TransactionalState.newCoordinatorState(conf, spoutId));
    }
    _currTransaction = getStoredCurrTransaction();

    _collector = collector;
    Number active = (Number) conf.get(Config.TOPOLOGY_MAX_SPOUT_PENDING);
    if(active==null) {
        _maxTransactionActive = 1;
    } else {
        _maxTransactionActive = active.intValue();
    }
    _attemptIds = getStoredCurrAttempts(_currTransaction, _maxTransactionActive);

    
    for(int i=0; i<_spouts.size(); i++) {
        String txId = _managedSpoutIds.get(i);
        _coordinators.add(_spouts.get(i).getCoordinator(txId, conf, context));
    }
}
 

import backtype.storm.spout.SpoutOutputCollector; //导入依赖的package包/类
@Override
public void open(Map conf, TopologyContext context,
		SpoutOutputCollector collector) {
	_collector = collector;
	if (_local_drpc_id == null) {
		int numTasks = context.getComponentTasks(
				context.getThisComponentId()).size();
		int index = context.getThisTaskIndex();

		int port = Utils.getInt(conf.get(Config.DRPC_INVOCATIONS_PORT));
		List<String> servers = (List<String>) conf.get(Config.DRPC_SERVERS);
		if (servers == null || servers.isEmpty()) {
			throw new RuntimeException(
					"No DRPC servers configured for topology");
		}
		if (numTasks < servers.size()) {
			for (String s : servers) {
				_clients.add(new DRPCInvocationsClient(s, port));
			}
		} else {
			int i = index % servers.size();
			_clients.add(new DRPCInvocationsClient(servers.get(i), port));
		}
	}

}
 

import backtype.storm.spout.SpoutOutputCollector; //导入依赖的package包/类
@Override
public void open(Map conf, TopologyContext context,
		SpoutOutputCollector collector) {
	_context = context;
	List<Integer> tasks = context.getComponentTasks(context
			.getThisComponentId());
	int startIndex;
	for (startIndex = 0; startIndex < tasks.size(); startIndex++) {
		if (tasks.get(startIndex) == context.getThisTaskId()) {
			break;
		}
	}
	_collector = collector;
	_pending = new HashMap<String, FixedTuple>();
	_serveTuples = new ArrayList<FixedTuple>();
	for (int i = startIndex; i < _tuples.size(); i += tasks.size()) {
		_serveTuples.add(_tuples.get(i));
	}
}
 

import backtype.storm.spout.SpoutOutputCollector; //导入依赖的package包/类
public void open(Map conf, TopologyContext context,
		SpoutOutputCollector collector) {
	_context = context;
	List<Integer> tasks = context.getComponentTasks(context
			.getThisComponentId());
	int startIndex;
	for (startIndex = 0; startIndex < tasks.size(); startIndex++) {
		if (tasks.get(startIndex) == context.getThisTaskId()) {
			break;
		}
	}
	_collector = collector;
	_pending = new HashMap<String, FixedTuple>();
	_serveTuples = new ArrayList<FixedTuple>();
	for (int i = startIndex; i < _tuples.size(); i += tasks.size()) {
		_serveTuples.add(_tuples.get(i));
	}
}
 

import backtype.storm.spout.SpoutOutputCollector; //导入依赖的package包/类
/**
 * {@inheritDoc}
 */
@Override
public void open(Map conf, TopologyContext context, SpoutOutputCollector collector)
{
    LOG.info("Start to open storm spout.");
    Map<String, IEmitter> emitters = createEmitters(collector);
    
    try
    {
        input.initialize(emitters);
    }
    catch (StreamingException e)
    {
        LOG.error("Failed to initialize input stream.");
        throw new RuntimeException("Failed to initialize output stream", e);
    }
    
}
 

import backtype.storm.spout.SpoutOutputCollector; //导入依赖的package包/类
@SuppressWarnings("rawtypes")
@Override
public void open(Map conf, TopologyContext context,
		SpoutOutputCollector collector) {
	batchQueue = new LinkedBlockingQueue<BatchSpoutMsgId>();
	this.collector = collector;
	this.conf = conf;
	taskName = context.getThisComponentId() + "_" + context.getThisTaskId();
	
	intervalCheck = new IntervalCheck();

	try {
		zkClient = BatchCommon.getZkClient(conf);

		initMsgId();

	} catch (Exception e) {
		LOG.error("", e);
		throw new RuntimeException("Failed to init");
	}
	LOG.info("Successfully open " + taskName);
}
 

import backtype.storm.spout.SpoutOutputCollector; //导入依赖的package包/类
public void open(Map config, TopologyContext context,
    SpoutOutputCollector collector) {
  this.m_collector = collector;
  this.m_dataset = Configuration.getDataSetSemEval2013();
  this.m_tweets = m_dataset.getTestTweets();

  // Optional sleep between tuples emitting
  if (config.get(CONF_TUPLE_SLEEP_MS) != null) {
    m_tupleSleepMs = (Long) config.get(CONF_TUPLE_SLEEP_MS);
  } else {
    m_tupleSleepMs = 0;
  }
  if (config.get(CONF_TUPLE_SLEEP_NS) != null) {
    m_tupleSleepNs = (Long) config.get(CONF_TUPLE_SLEEP_NS);
  } else {
    m_tupleSleepNs = 0;
  }

  // Optional startup sleep to finish bolt preparation
  // before spout starts emitting
  if (config.get(CONF_STARTUP_SLEEP_MS) != null) {
    long startupSleepMillis = (Long) config.get(CONF_STARTUP_SLEEP_MS);
    TimeUtils.sleepMillis(startupSleepMillis);
  }
}