Python storm.emit函数代码示例

 def nextTuple(self):
     try:
         input_tuple = None
         try:
             if self.counter >= self.script._num_iterations:
                 return
         except:
             try:
                 input_tuple = self.script._static_input.pop(0)
             except AttributeError:
                 # there is no static input
                 pass
             except IndexError:
                 # static input is empty - no more processing
                 return
         outputs = self.script.process(input_tuple)
         if outputs is None:
             return
         for streamname, output in outputs.iteritems():
             result = output if isinstance(output, list) else [output]
             storm.emit(result, stream=streamname, id=self.counter)
             storm.log("Dispel4Py ------> %s: emitted tuple %s to stream %s" % (self.script.id, result, streamname))
             self.counter += 1
     except:
         # logging the error but it should be passed to client somehow
         storm.log("Dispel4Py ------> %s: %s" % (self.scriptname, traceback.format_exc(), ))

 def process(self, tup):
     #TO DO: Add check for empty values
     if tup.values[0]:
         words = tup.values[0].split(" ")
         if words:
             for word in words:
                 storm.emit([word])

 def process(self,tup):
     if tup.values:
         words = tup.values[0]
         if words:
             storm.emit([words])
     else:
         pass

 def process(self, tup):
   text = tup.values[1]
   #language = langid.classify(text)[0]
   #l = LangID()
   #l.train()
   language = self.l.classify(text)
   storm.emit([tup.values[0], language])

    def process(self, tup):
        segments = tup.values[1].rstrip('/').rsplit('/', 1)
        path = segments[0] if self.match(segments[-1]) else '/'.join(segments)

        event = dict(
            timestamp=tup.values[0],
            path=path
            )

        kwargs = dict(
            id=tup.values[2]
            )

        try:
            # TODO: Retrieve users from all indicies.
            events = self.es.get(self.index, kwargs['id'], 'user',
                                 preference='_primary')
            kwargs['version'] = events['_version']
            body = {'events': events['_source']['events'] + [event]}
        except NotFoundError:
            kwargs['op_type'] = 'create'
            body = {'events': [event]}

        try:
            body['rank'] = math.log10(len(body['events'])) / 2
            self.es.index(self.index, 'user', body, **kwargs)
            paths = list(event['path'] for event in body['events'])
            emit([kwargs['id'], paths])
            ack(tup)
        except TransportError:
            fail(tup)

 def process(self, tup):
     url = tup.values[0]
     storm.log("HARing "+url)
     output = self.get_har_with_image(url)
     if output is not "FAIL":
         storm.emit(output, anchors=[tup])
         storm.ack(tup)

    def process(self, tup):
        sentence = tup.values[0]
        sentence = re.sub(r"[,.;!\?]", "", sentence) # get rid of punctuation and num
        words = jieba.cut(sentence, cut_all=True)

        for word in words:
            storm.emit([word])

  def emit(self):

    if not self.entity:
      self.entity = self.getLastEntity()

    # We always emit tuple = (entity, "{json string}")
    storm.emit([self.entity,json.dumps(self.d)])

    def nextTuple(self):
        try: 
            html = urllib2.urlopen(self.url_animals_tl).read()
            soup = BeautifulSoup(html)
            scripts = soup.find_all('script')
            code = scripts[len(scripts) - 1]

            content = code.contents[0].strip()
            prefix = 'P.start.start('
            i = content.find(prefix)
            if i != -1:
                json_str = content[i + len(prefix) : len(content) - 2]
                json_obj = json.loads(json_str)
                pins = json_obj['tree']['children'][3]['children'][0]['children'][0]['children']
                
                for pin in pins:
                    pin_id = pin['options']['pin_id']
                    if 'module' in pin['children'][1]['options']:
                        module = pin['children'][1]['options']['module']
                    orig_link = pin['data']['link']
                    orig_host = urlparse(pin['data']['link']).hostname
                    images = pin['data']['images']
                    if 'orig' in images:
                        pass
                    storm.emit([pin_id, orig_link, orig_host, json.dumps(pin, indent=4, sort_keys=True), self.category])
                
            time.sleep(2)
        except StopIteration:
            pass
        except urllib2.HTTPError, err:
            if err.code == 404:
                pass

	def process(self, tup):
		sentence = tup.values[0].lower()
		words = sentence.split(" ")
		bow = numpy.zeros((self._n,))
		for w in words:
			bow[self.map[w]]+=1   	
		storm.emit([bow.tolist()])

 def process(self, tup):
     word = tup.values[0];
     if (random() < 0.75):
         storm.emit([word + 'lalala'], anchors=[tup])
         storm.ack(tup)
     else:
         storm.log(word + ' randomly skipped!')

 def process(self,tup):
     # storm.log("Dispel4Py ------> %s: Received block." % (self.script.id, ))
     try:
         inputname = self.inputmapping[tup.component][tup.stream]
         storm.log("Dispel4Py ------> %s: Received block at input '%s'" % (self.script.id, inputname, ))
         # inputs = tup.values
         inputs = decode_types(tup.values)
         outputs = self.script.process( { inputname : inputs })
         # storm.log("Dispel4Py ------> %s: Processing complete." % self.scriptname)
         
         if outputs is None:
             return
         for streamname, output in outputs.iteritems():
             result = output if isinstance(output, list) else [output]
             try:
                 storm.emit(result, stream=streamname)
                 storm.log("Dispel4Py ------> %s: Emitted to stream %s: %s" % (self.script.id, streamname, str(result)[:200]))
             except TypeError:
                 # encode manually
                 encoded = encode_types(result)
                 storm.emit(encoded, stream=streamname)
                 storm.log("Dispel4Py ------> %s: Emitted to stream %s" % (self.script.id, streamname))
             # except:
             #     storm.log("%s: %s" % (self.script.id, traceback.format_exc()))
     except:
         storm.log("Dispel4Py ------> %s: %s" % (self.script.id, traceback.format_exc(), ))

 def nextTuple(self):
     docs = self.get_docs()
     uuid = docs[0]['uuid']
     tup = [docs[0]['href'][18:]]
     self.buffer[uuid] = (tup, 0)
     emit(tup, id=uuid)
     sleep(1.0)

 def nextTuple(self):
     id = "periodic_%s" % str(uuid4())
     body = json.dumps({'message_id': CHECK_METRIC_ALARM_MSG_ID})
     message = "Periodic monitoring message sent [%s] %s"
     self.log(message % (id, body))
     emit([None, body], id=id)
     time.sleep(60)

    def process(self, tuple):
        '''
        Must fulfil the following contract expressed in the Java wrapper:

        declarer.declare(new Fields(TopologyFields.AUTHOR_SCREEN_NAME, TopologyFields.CREATED_AT,
                TopologyFields.FAV_COUNT, TopologyFields.HASHTAGS_TEXTS, TopologyFields.IN_REPLY_TO_SCREEN_NAME, 
                TopologyFields.LANG, TopologyFields.RETWEET_COUNT, TopologyFields.RETWEETED, 
                TopologyFields.SOURCE, TopologyFields.PLACE, TopologyFields.POSSIBLY_SENSITIVE,
                TopologyFields.TEXT, TopologyFields.TOPIC_NAME));
        '''
        place, topic_name, query = tuple.values
        try: 
            tweets = list(get_tweets.get_tweets_for_trends(self._twitter_api, [{"query" : query}], popular = True, tweet_processor = self._storm_tweet_processor))[0]["tweets"]
        except tweepy.TweepError as te:
            # We have hit the REST API Rate limit for Twitter https://dev.twitter.com/docs/rate-limiting/1.1, no more tweets for some time
            log_tweeter_error(te, sleep_time=self._rate_limit_sleep_time)
            return 

        for pt in tweets:
            # Here we add the trending topic name, and take the place name from those
            # used internally by get_tweets, instead of the from place names returned by twitter
            tup = [pt['author_screen_name'], pt['created_at'], 
                   pt['favorite_count'], pt['hashtags_texts'], pt['in_reply_to_screen_name'],
                   pt['lang'], pt['retweet_count'], pt['retweeted'],
                   pt['source'], place, pt['possibly_sensitive'],
                   pt['text'], topic_name]
            storm.emit(tup)