本文整理汇总了Python中storm.emit函数的典型用法代码示例。如果您正苦于以下问题:Python emit函数的具体用法?Python emit怎么用?Python emit使用的例子?那么恭喜您, 这里精选的函数代码示例或许可以为您提供帮助。
在下文中一共展示了emit函数的15个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: nextTuple
def nextTuple(self):
try:
input_tuple = None
try:
if self.counter >= self.script._num_iterations:
return
except:
try:
input_tuple = self.script._static_input.pop(0)
except AttributeError:
# there is no static input
pass
except IndexError:
# static input is empty - no more processing
return
outputs = self.script.process(input_tuple)
if outputs is None:
return
for streamname, output in outputs.iteritems():
result = output if isinstance(output, list) else [output]
storm.emit(result, stream=streamname, id=self.counter)
storm.log("Dispel4Py ------> %s: emitted tuple %s to stream %s" % (self.script.id, result, streamname))
self.counter += 1
except:
# logging the error but it should be passed to client somehow
storm.log("Dispel4Py ------> %s: %s" % (self.scriptname, traceback.format_exc(), ))
示例2: process
def process(self, tup):
#TO DO: Add check for empty values
if tup.values[0]:
words = tup.values[0].split(" ")
if words:
for word in words:
storm.emit([word])
示例3: process
def process(self,tup):
if tup.values:
words = tup.values[0]
if words:
storm.emit([words])
else:
pass
示例4: process
def process(self, tup):
text = tup.values[1]
#language = langid.classify(text)[0]
#l = LangID()
#l.train()
language = self.l.classify(text)
storm.emit([tup.values[0], language])
示例5: process
def process(self, tup):
segments = tup.values[1].rstrip('/').rsplit('/', 1)
path = segments[0] if self.match(segments[-1]) else '/'.join(segments)
event = dict(
timestamp=tup.values[0],
path=path
)
kwargs = dict(
id=tup.values[2]
)
try:
# TODO: Retrieve users from all indicies.
events = self.es.get(self.index, kwargs['id'], 'user',
preference='_primary')
kwargs['version'] = events['_version']
body = {'events': events['_source']['events'] + [event]}
except NotFoundError:
kwargs['op_type'] = 'create'
body = {'events': [event]}
try:
body['rank'] = math.log10(len(body['events'])) / 2
self.es.index(self.index, 'user', body, **kwargs)
paths = list(event['path'] for event in body['events'])
emit([kwargs['id'], paths])
ack(tup)
except TransportError:
fail(tup)
示例6: process
def process(self, tup):
url = tup.values[0]
storm.log("HARing "+url)
output = self.get_har_with_image(url)
if output is not "FAIL":
storm.emit(output, anchors=[tup])
storm.ack(tup)
示例7: process
def process(self, tup):
sentence = tup.values[0]
sentence = re.sub(r"[,.;!\?]", "", sentence) # get rid of punctuation and num
words = jieba.cut(sentence, cut_all=True)
for word in words:
storm.emit([word])
示例8: emit
def emit(self):
if not self.entity:
self.entity = self.getLastEntity()
# We always emit tuple = (entity, "{json string}")
storm.emit([self.entity,json.dumps(self.d)])
示例9: nextTuple
def nextTuple(self):
try:
html = urllib2.urlopen(self.url_animals_tl).read()
soup = BeautifulSoup(html)
scripts = soup.find_all('script')
code = scripts[len(scripts) - 1]
content = code.contents[0].strip()
prefix = 'P.start.start('
i = content.find(prefix)
if i != -1:
json_str = content[i + len(prefix) : len(content) - 2]
json_obj = json.loads(json_str)
pins = json_obj['tree']['children'][3]['children'][0]['children'][0]['children']
for pin in pins:
pin_id = pin['options']['pin_id']
if 'module' in pin['children'][1]['options']:
module = pin['children'][1]['options']['module']
orig_link = pin['data']['link']
orig_host = urlparse(pin['data']['link']).hostname
images = pin['data']['images']
if 'orig' in images:
pass
storm.emit([pin_id, orig_link, orig_host, json.dumps(pin, indent=4, sort_keys=True), self.category])
time.sleep(2)
except StopIteration:
pass
except urllib2.HTTPError, err:
if err.code == 404:
pass
示例10: process
def process(self, tup):
sentence = tup.values[0].lower()
words = sentence.split(" ")
bow = numpy.zeros((self._n,))
for w in words:
bow[self.map[w]]+=1
storm.emit([bow.tolist()])
示例11: process
def process(self, tup):
word = tup.values[0];
if (random() < 0.75):
storm.emit([word + 'lalala'], anchors=[tup])
storm.ack(tup)
else:
storm.log(word + ' randomly skipped!')
示例12: process
def process(self,tup):
# storm.log("Dispel4Py ------> %s: Received block." % (self.script.id, ))
try:
inputname = self.inputmapping[tup.component][tup.stream]
storm.log("Dispel4Py ------> %s: Received block at input '%s'" % (self.script.id, inputname, ))
# inputs = tup.values
inputs = decode_types(tup.values)
outputs = self.script.process( { inputname : inputs })
# storm.log("Dispel4Py ------> %s: Processing complete." % self.scriptname)
if outputs is None:
return
for streamname, output in outputs.iteritems():
result = output if isinstance(output, list) else [output]
try:
storm.emit(result, stream=streamname)
storm.log("Dispel4Py ------> %s: Emitted to stream %s: %s" % (self.script.id, streamname, str(result)[:200]))
except TypeError:
# encode manually
encoded = encode_types(result)
storm.emit(encoded, stream=streamname)
storm.log("Dispel4Py ------> %s: Emitted to stream %s" % (self.script.id, streamname))
# except:
# storm.log("%s: %s" % (self.script.id, traceback.format_exc()))
except:
storm.log("Dispel4Py ------> %s: %s" % (self.script.id, traceback.format_exc(), ))
示例13: nextTuple
def nextTuple(self):
docs = self.get_docs()
uuid = docs[0]['uuid']
tup = [docs[0]['href'][18:]]
self.buffer[uuid] = (tup, 0)
emit(tup, id=uuid)
sleep(1.0)
示例14: nextTuple
def nextTuple(self):
id = "periodic_%s" % str(uuid4())
body = json.dumps({'message_id': CHECK_METRIC_ALARM_MSG_ID})
message = "Periodic monitoring message sent [%s] %s"
self.log(message % (id, body))
emit([None, body], id=id)
time.sleep(60)
示例15: process
def process(self, tuple):
'''
Must fulfil the following contract expressed in the Java wrapper:
declarer.declare(new Fields(TopologyFields.AUTHOR_SCREEN_NAME, TopologyFields.CREATED_AT,
TopologyFields.FAV_COUNT, TopologyFields.HASHTAGS_TEXTS, TopologyFields.IN_REPLY_TO_SCREEN_NAME,
TopologyFields.LANG, TopologyFields.RETWEET_COUNT, TopologyFields.RETWEETED,
TopologyFields.SOURCE, TopologyFields.PLACE, TopologyFields.POSSIBLY_SENSITIVE,
TopologyFields.TEXT, TopologyFields.TOPIC_NAME));
'''
place, topic_name, query = tuple.values
try:
tweets = list(get_tweets.get_tweets_for_trends(self._twitter_api, [{"query" : query}], popular = True, tweet_processor = self._storm_tweet_processor))[0]["tweets"]
except tweepy.TweepError as te:
# We have hit the REST API Rate limit for Twitter https://dev.twitter.com/docs/rate-limiting/1.1, no more tweets for some time
log_tweeter_error(te, sleep_time=self._rate_limit_sleep_time)
return
for pt in tweets:
# Here we add the trending topic name, and take the place name from those
# used internally by get_tweets, instead of the from place names returned by twitter
tup = [pt['author_screen_name'], pt['created_at'],
pt['favorite_count'], pt['hashtags_texts'], pt['in_reply_to_screen_name'],
pt['lang'], pt['retweet_count'], pt['retweeted'],
pt['source'], place, pt['possibly_sensitive'],
pt['text'], topic_name]
storm.emit(tup)