Python amqp.consume_items函数代码示例

def process_adzerk():
    @g.stats.amqp_processor('adzerk_q')
    def _handle_adzerk(msg):
        data = json.loads(msg.body)
        g.log.debug('data: %s' % data)

        action = data.get('action')

        if action == 'deactivate_orphaned_flight':
            _deactivate_orphaned_flight(data['flight'])
            return

        link = Link._by_fullname(data['link'], data=True)
        if data['campaign']:
            campaign = PromoCampaign._by_fullname(data['campaign'], data=True)
        else:
            campaign = None

        if action == 'update_adzerk':
            if 'triggered_by' in data and data['triggered_by'] is not None:
                triggered_by = Account._by_fullname(data['triggered_by'], data=True)
            else:
                triggered_by = None

            _update_adzerk(link, campaign, triggered_by)

        elif action == 'deactivate_overdelivered':
            _deactivate_overdelivered(link, campaign)

    amqp.consume_items('adzerk_q', _handle_adzerk, verbose=False)

def process_liveupdate_scraper_q():
    @g.stats.amqp_processor('liveupdate_scraper_q')
    def _handle_q(msg):
        d = json.loads(msg.body)

        try:
            fn = TimeoutFunction(parse_embeds, 10)
            liveupdate = fn(d['event_id'], d['liveupdate_id'])
        except TimeoutFunctionException:
            g.log.warning(
                "Timed out on %s::%s", d["event_id"], d["liveupdate_id"])
            return
        except Exception as e:
            g.log.warning("Failed to scrape %s::%s: %r",
                d["event_id"], d["liveupdate_id"], e)
            return

        payload = {
            "liveupdate_id": "LiveUpdate_" + d['liveupdate_id'],
            "media_embeds": liveupdate.embeds,
            "mobile_embeds": liveupdate.mobile_embeds,
        }
        send_event_broadcast(d['event_id'],
                             type="embeds_ready",
                             payload=payload)

    amqp.consume_items('liveupdate_scraper_q', _handle_q, verbose=False)

def process_votes(qname, limit=0):
    # limit is taken but ignored for backwards compatibility

    @g.stats.amqp_processor(qname)
    def _handle_vote(msg):
        timer = stats.get_timer("service_time." + qname)
        timer.start()

        #assert(len(msgs) == 1)
        r = pickle.loads(msg.body)

        uid, tid, dir, ip, organic, cheater = r
        voter = Account._byID(uid, data=True)
        votee = Thing._by_fullname(tid, data = True)
        timer.intermediate("preamble")

        if isinstance(votee, Comment):
            update_comment_votes([votee])
            timer.intermediate("update_comment_votes")

        # I don't know how, but somebody is sneaking in votes
        # for subreddits
        if isinstance(votee, (Link, Comment)):
            print (voter, votee, dir, ip, organic, cheater)
            handle_vote(voter, votee, dir, ip, organic,
                        cheater = cheater, foreground=True, timer=timer)

    amqp.consume_items(qname, _handle_vote, verbose = False)

def run():
    @g.stats.amqp_processor("butler_q")
    def process_message(msg):
        fname = msg.body
        item = Thing._by_fullname(fname, data=True)
        monitor_mentions(item)

    amqp.consume_items("butler_q", process_message, verbose=True)

def run_summary_email_q(verbose=False):
    queue_name = 'summary_email_q'

    @g.stats.amqp_processor(queue_name)
    def _run_summary_email(msg):
        account_thing_id = int(msg.body)
        send_account_summary_email(account_thing_id)

    amqp.consume_items(queue_name, _run_summary_email, verbose=verbose)

def consume_vote_queue(queue):
    @g.stats.amqp_processor(queue)
    def process_message(msg):
		# msg is *PROBABLY* json
        timer = g.stats.get_timer("new_voting.%s" % queue)
        timer.start()

		# json being loaded into a python object
		# it has the fields "user_id", "thing_fullname"
		# a thing is a database object
		# it's a link, comment, post, whatever, everything can be upvoted/downvoted
        vote_data = json.loads(msg.body)
        hook = hooks.get_hook('vote.validate_vote_data')
        if hook.call_until_return(msg=msg, vote_data=vote_data) is False:
            # Corrupt records in the queue. Ignore them.
            print "Ignoring invalid vote by %s on %s %s" % (
                    vote_data.get('user_id', '<unknown>'),
                    vote_data.get('thing_fullname', '<unknown>'),
                    vote_data)
            return

		# this gets the user from database/cache (either memcached or postgres, whatever)
        user = Account._byID(vote_data.pop("user_id"), data=True)
        thing = Thing._by_fullname(vote_data.pop("thing_fullname"), data=True)

        timer.intermediate("preamble")

		# this gets a servers-wide lock
		# I mean, a bunch of consumers might be consuming items that use the same "thing" (same database object)
		# so, you want a global lock to avoid them from fucking eachother up 
		# memcachd stores the lock, atomically
        lock_key = "vote-%s-%s" % (user._id36, thing._fullname)
        with g.make_lock("voting", lock_key, timeout=5):
            print "Processing vote by %s on %s %s" % (user, thing, vote_data)

            try:
                vote = Vote(
                    user,
                    thing,
                    direction=vote_data["direction"],
                    date=datetime.utcfromtimestamp(vote_data["date"]),
                    data=vote_data["data"],
                    event_data=vote_data.get("event_data"),
                )
            except TypeError as e:
                # a vote on an invalid type got in the queue, just skip it
                g.log.exception("Invalid type: %r", e.message)
                return

            timer.intermediate("create_vote_obj")

            vote.commit()

            timer.flush()

    amqp.consume_items(queue, process_message, verbose=False)

def consume_link_vote_queue(qname="vote_link_q"):
    @g.stats.amqp_processor(qname)
    def process_message(msg):
        vote_data = json.loads(msg.body)
        hook = hooks.get_hook('vote.validate_vote_data')
        if hook.call_until_return(msg=msg, vote_data=vote_data) is False:
            # Corrupt records in the queue. Ignore them.
            print "Ignoring invalid vote by %s on %s %s" % (
                    vote_data.get('user_id', '<unknown>'),
                    vote_data.get('thing_fullname', '<unknown>'),
                    vote_data)
            return

        timer = g.stats.get_timer("link_vote_processor")
        timer.start()

        user = Account._byID(vote_data.pop("user_id"))
        link = Link._by_fullname(vote_data.pop("thing_fullname"))

        # create the vote and update the voter's liked/disliked under lock so
        # that the vote state and cached query are consistent
        lock_key = "vote-%s-%s" % (user._id36, link._fullname)
        with g.make_lock("voting", lock_key, timeout=5):
            print "Processing vote by %s on %s %s" % (user, link, vote_data)

            try:
                vote = Vote(
                    user,
                    link,
                    direction=vote_data["direction"],
                    date=datetime.utcfromtimestamp(vote_data["date"]),
                    data=vote_data["data"],
                    event_data=vote_data.get("event_data"),
                )
            except TypeError as e:
                # a vote on an invalid type got in the queue, just skip it
                g.log.exception("Invalid type: %r", e.message)
                return

            vote.commit()
            timer.intermediate("create_vote_object")

            update_user_liked(vote)
            timer.intermediate("voter_likes")

        vote_valid = vote.is_automatic_initial_vote or vote.effects.affects_score
        link_valid = not (link._spam or link._deleted)
        if vote_valid and link_valid:
            add_to_author_query_q(link)
            add_to_subreddit_query_q(link)
            add_to_domain_query_q(link)

        timer.stop()
        timer.flush()

    amqp.consume_items(qname, process_message, verbose=False)

def run_subreddit_maker():
    @g.stats.amqp_processor('robin_subreddit_maker_q')
    def process_subreddit_maker(msg):
        room_id = msg.body
        try:
            room = RobinRoom._byID(room_id)
        except tdb_cassandra.NotFound:
            try:
                room = RobinRoomDead._byID(room_id)
            except tdb_cassandra.NotFound:
                print "can't find room %s, giving up" % room_id
        print 'creating sr for room %s' % room

        subreddit = room.create_sr()
        print 'got %s from room.create_sr()' % subreddit

        if subreddit:
            g.stats.simple_event("robin.subreddit.created")
            participant_ids = room.get_all_participants()
            participants = [
                Account._byID(participant_id)
                for participant_id in participant_ids
            ]
            moderators = participants[:5]

            print 'adding moderators to %s' % subreddit
            for moderator in moderators:
                subreddit.add_moderator(moderator)

            print 'adding contributors to %s' % subreddit
            g.stats.simple_event(
                "robin.subreddit.contributors_added",
                delta=len(participants),
            )
            for participant in participants:
                # To be replaced with UserRel hacking?
                subreddit.add_contributor(participant)
                send_sr_message(subreddit, participant)

            payload = {
                "body": subreddit.name,
            }

            websockets.send_broadcast(
                namespace="/robin/" + room.id,
                type="continue",
                payload=payload,
            )
        else:
            g.stats.simple_event("robin.subreddit.creation_failed")
            print 'subreddit creation failed for room %s' % room.id

    amqp.consume_items('robin_subreddit_maker_q', process_subreddit_maker)

def run_new_comments():
    """Add new incoming comments to the /comments page"""
    # this is done as a queue because otherwise the contention for the
    # lock on the query would be very high

    def _run_new_comment(msg):
        fname = msg.body
        comment = Comment._by_fullname(fname)

        add_queries([get_all_comments()], insert_items=[comment])

    amqp.consume_items("newcomments_q", _run_new_comment)

def consume_vote_queue(queue):
    @g.stats.amqp_processor(queue)
    def process_message(msg):
        timer = g.stats.get_timer("new_voting.%s" % queue)
        timer.start()

        vote_data = json.loads(msg.body)
        hook = hooks.get_hook('vote.validate_vote_data')
        if hook.call_until_return(msg=msg, vote_data=vote_data) is False:
            # Corrupt records in the queue. Ignore them.
            print "Ignoring invalid vote by %s on %s %s" % (
                    vote_data.get('user_id', '<unknown>'),
                    vote_data.get('thing_fullname', '<unknown>'),
                    vote_data)
            return

        # if it's an old-style vote, convert to the new format
        if "uid" in vote_data:
            vote_data = convert_old_vote_data(vote_data, msg.timestamp)

        user = Account._byID(vote_data.pop("user_id"), data=True)
        thing = Thing._by_fullname(vote_data.pop("thing_fullname"), data=True)

        timer.intermediate("preamble")

        lock_key = "vote-%s-%s" % (user._id36, thing._fullname)
        with g.make_lock("voting", lock_key, timeout=5):
            print "Processing vote by %s on %s %s" % (user, thing, vote_data)

            try:
                vote = Vote(
                    user,
                    thing,
                    direction=vote_data["direction"],
                    date=datetime.utcfromtimestamp(vote_data["date"]),
                    data=vote_data["data"],
                    event_data=vote_data.get("event_data"),
                )
            except TypeError as e:
                # a vote on an invalid type got in the queue, just skip it
                g.log.exception("Invalid type: %r", e.message)
                return

            timer.intermediate("create_vote_obj")

            vote.commit()

            timer.flush()

    amqp.consume_items(queue, process_message, verbose=False)

def run():
    @g.stats.amqp_processor("automoderator_q")
    def process_message(msg):
        if not ACCOUNT:
            return

        fullname = msg.body
        item = Thing._by_fullname(fullname, data=True)
        if not isinstance(item, (Link, Comment)):
            return

        subreddit = item.subreddit_slow

        wiki_page_id = wiki_id(subreddit._id36, "config/automoderator")
        wiki_page_fullname = "WikiPage_%s" % wiki_page_id
        last_edited = LastModified.get(wiki_page_fullname, "Edit")
        if not last_edited:
            return

        # initialize rules for the subreddit if we haven't already
        # or if the page has been edited since we last initialized
        need_to_init = False
        if subreddit._id not in rules_by_subreddit:
            need_to_init = True
        else:
            rules = rules_by_subreddit[subreddit._id]
            if last_edited > rules.init_time:
                need_to_init = True

        if need_to_init:
            wp = WikiPage.get(subreddit, "config/automoderator")
            rules = Ruleset(wp.content)
            rules_by_subreddit[subreddit._id] = rules

        if not rules:
            return

        try:
            TimeoutFunction(rules.apply_to_item, 2)(item)
            print "Checked %s from /r/%s" % (item, subreddit.name)
        except TimeoutFunctionException:
            print "Timed out on %s from /r/%s" % (item, subreddit.name)
        except KeyboardInterrupt:
            raise
        except:
            print "Error on %s from /r/%s" % (item, subreddit.name)
            print traceback.format_exc()

    amqp.consume_items("automoderator_q", process_message, verbose=False)

def run_new_comments():
    """Add new incoming comments to the /comments page"""
    # this is done as a queue because otherwise the contention for the
    # lock on the query would be very high

    def _run_new_comment(msg):
        fname = msg.body
        comment = Comment._by_fullname(fname,data=True)
        sr = Subreddit._byID(comment.sr_id)

        add_queries([get_all_comments(),
                     get_sr_comments(sr)],
                    insert_items = [comment])

    amqp.consume_items('newcomments_q', _run_new_comment)

def process_votes(limit=None):
    # limit is taken but ignored for backwards compatibility

    def _handle_vote(msg):
        r = pickle.loads(msg.body)

        uid, tid, dir, ip, organic, cheater = r
        voter = Account._byID(uid, data=True)
        votee = Thing._by_fullname(tid, data = True)

        print (voter, votee, dir, ip, organic, cheater)
        handle_vote(voter, votee, dir, ip, organic,
                    cheater = cheater)

    amqp.consume_items('register_vote_q', _handle_vote)

def run():
    # Add watch to only update kwl when the keyword list changes
    @g.zookeeper.DataWatch("/keyword-targets")
    def watch_keywords(data, stats):
        global kwl
        kwl = json.loads(data)
        
    @g.stats.amqp_processor("keyword_target_q")
    def process_message(msg):
        fname = msg.body
        link = Link._by_fullname(fname, data=True)
        extract_keywords(link)

    amqp.consume_items("keyword_target_q",
                       process_message,
                       verbose=True)

def process():
    processor = Processor()
    processor.register("upsert_promotion", _handle_upsert_promotion)
    processor.register("upsert_campaign", _handle_upsert_campaign)
    processor.register("deactivate_campaign", _handle_deactivate_campaign)

    @g.stats.amqp_processor(DFP_QUEUE)
    def _handler(message):
        data = json.loads(message.body)
        g.log.debug("processing action: %s" % data)

        action = data.get("action")
        payload = data.get("payload")

        processor.call(action, payload)

    amqp.consume_items(DFP_QUEUE, _handler, verbose=False)