Python TwitterSearch.get_statistics方法代码示例

# 需要导入模块: import TwitterSearch [as 别名]
# 或者: from TwitterSearch import get_statistics [as 别名]
from TwitterSearch import *
from settings import *
import time


try:
    tso = TwitterSearchOrder()
    tso.set_keywords(['brexit', 'May'])
    
    ts = TwitterSearch(
                       consumer_key = twitter_consumer_key,
                       consumer_secret = twitter_consumer_secret,
                       access_token = twitter_access_token,
                       access_token_secret = twitter_access_token_secret
                       )
    sleep_for = 5 # sleep for 60 seconds
    last_amount_of_queries = 0 # used to detect when new queries are done
    
    for tweet in ts.search_tweets_iterable(tso):
        print tweet
#        print( '@%s tweeted: %s' % ( tweet['user']['screen_name'], tweet['text'] ) )
        current_amount_of_queries = ts.get_statistics()[0]
        if not last_amount_of_queries == current_amount_of_queries:
            last_amount_of_queries = current_amount_of_queries
            time.sleep(sleep_for)



except TwitterSearchException as e:
    print(e)

# 需要导入模块: import TwitterSearch [as 别名]
# 或者: from TwitterSearch import get_statistics [as 别名]
last_num_queries = 0
retrieved_tweets = set()
number_of_repeat_tweets = 0

try:
    tweetpkl = open('lol.pkl','wb')
    tweets = []
    for tweet in ts.search_tweets_iterable(tso):
        tweets.append(tweet)
        if tweet['id'] in retrieved_tweets:
            print("tweet has already been retrieved")
            number_of_repeat_tweets += 1
        else:
            retrieved_tweets.add(tweet['id'])

        cur_num_queries,tweets_recd = ts.get_statistics()
        if not last_num_queries == cur_num_queries:
            print("Queries done: %i. Tweets received: %i" % ts.get_statistics())
            last_num_queries = cur_num_queries
            print("Sleeping for",sleep_for,"seconds")
            time.sleep(sleep_for)

    pickle.dump(tweets,tweetpkl)
    print("Wrote tweets to file",tweetpkl.name)
    print("number_of_repeat_tweets: ",number_of_repeat_tweets)
    tweetpkl.close()

except TwitterSearchException as e:
    if e.code < 1000:
        print("HTTP status based exception: %i - %s" % (e.code, e.message))
    else:

# 需要导入模块: import TwitterSearch [as 别名]
# 或者: from TwitterSearch import get_statistics [as 别名]
def twitSearch(tweetLastSeen):
    #print("Debug: In function twitSearch()")
    tweetSearchCount = 0
    try:
        tso = TwitterSearchOrder()
        #tso.set_keywords(['disaster','banking'], or_operator = True)
        tso.set_keywords(['disaster','poverty','banking','homeless'], or_operator = True)
        #tso.add_keyword('poverty')
        #tso.add_keyword('disaster')
        #tso.add_keyword('banking')
        tso.set_language('en')
        tso.set_include_entities(False)
        tso.set_result_type('recent')

        if tweetLastSeen > 0:
            print("Debug: I have a previous value for lastseen_id, setting since_id() to: %s and asking for 100 results" % tweetLastSeen)
            tso.set_since_id(tweetLastSeen)
            tso.set_count(100)
        else:
            print("Debug: No value for lastseen_id, asking for one result")
            tso.set_count(1)

        print("Debug: The tso search string looks like this")
        print(tso.create_search_url())

        ts = TwitterSearch(
            consumer_key = '',
            consumer_secret = '',
            access_token = '',
            access_token_secret = '')

##        def my_callback_function(current_ts_instance): # accepts ONE argument: an instance of TwitterSearch
##            #print("In callback function")
##            queries, tweets_seen = current_ts_instance.get_statistics()
##            #query = current_ts_instance.get_statistics()
##            print("%s queries & %s tweets seen" %(queries, tweets_seen))
##            print("%s query" %(query))
##            #if queries > 0 and (queries % 5) == 0: # trigger delay every 5th query
##                #print("Thats 5 queries. Sleeping for 60 secs")
##                #time.sleep(60) # sleep for 60 seconds

        #queries, tweets_seen = ts.get_statistics()
        #print("Debug: %s queries & %s tweets seen" %(queries, tweets_seen))

        #print("Debug: About to iterate over search results from TwitterSearch instance")
        #for tweet in ts.search_tweets_iterable(tso, callback=my_callback_function):
                
        tweets_seen = 0        
        currentTweetID = 0
        lastTweetID = 0

        for tweet in ts.search_tweets_iterable(tso):    
            queries, tweets_seen_by_stats = ts.get_statistics()
            print("Debug: stats: %s queries & %s tweets seen" %(queries, tweets_seen_by_stats))
            rateLimitRemaining = ts.get_metadata()['x-rate-limit-remaining']
            rateLimitReset = ts.get_metadata()['X-Rate-Limit-Reset']
            print("Debug: Rate limit resets at %s and has %s queries remaining" %(datetime.datetime.fromtimestamp(int(rateLimitReset)), rateLimitRemaining))
            currentTweetID = tweet['id']
            print("Debug: Current tweetID %s" % currentTweetID)
            if currentTweetID > lastTweetID:
                print("Debug: Seen a more recent tweetID, updating lastTweetID")
                lastTweetID = currentTweetID
                tweets_seen = tweets_seen_by_stats
                break
            print( 'Debug: In tweet iter @%s tweet id: %s' % ( tweet['user']['screen_name'], tweet['id'] ) )
            tweets_seen = tweets_seen + 1
            print("Debug: tweets_seens: %s" % tweets_seen)
            
        print('Debug: about to return tweet ID @%s' % lastTweetID )
        global twitterSearchCount
        twitterSearchCount = twitterSearchCount + 1
        print("Debug: This is twitter search number: %s" % twitterSearchCount)
        
        return lastTweetID, tweets_seen

    except TwitterSearchException as e:
        print(e)

# 需要导入模块: import TwitterSearch [as 别名]
# 或者: from TwitterSearch import get_statistics [as 别名]

#.........这里部分代码省略.........
                embedded_urls = tweet["entities"]["urls"]
                for url in embedded_urls:
                    url_statement = u"""INSERT INTO `embedded_urls`(
                    `tweet_id`
                    , `url`
                    , `start`
                    , `end`)
                     VALUES (%s,%s,%s,%s)
                     ON DUPLICATE KEY UPDATE tweet_id=tweet_id
                    """
                    self.cursor.execute(url_statement, (
                        tweet["id_str"],
                        url["url"],
                        url["indices"][0],
                        url["indices"][1]
                    ))

                # TODO: Investigate crash due tolack of x-rate-limit-remaining key
                if 'x-rate-limit-remaining' in self.ts.get_metadata():
                    self.log("\nCurrent rate-limiting status: %s" %
                             self.ts.get_metadata()['x-rate-limit-remaining'])

                # self.log('\[email protected]%s tweeted: %s'
                # % (tweet['user']['screen_name'], tweet['text']))

                self.add_queries_to_DB(new_queries)

                # Execute queries for real
                self.con.commit()

                # if current_num_of_queries is different
                # from last_num_of_queries,
                # then a new page was loaded by TwitterSearch
                current_num_of_queries = self.ts.get_statistics()[0]
                if not last_num_of_queries == current_num_of_queries:
                    # Calculate current query rate
                    now = int(time.time())
                    if('x-rate-limit-remaining' in self.ts.get_metadata()):
                        reset = int(self.ts.get_metadata()['x-rate-limit-reset'])
                    else:
                        reset = 0

                    if((now - start) == 0):
                        rate = this_query_total_tweets
                    else:
                        rate = this_query_total_tweets / (now - start)

                    self.log("\nCurrent API query rate: %s queries / s" % rate)
                    self.log("\nCUrrent tweets processed: %s" % self.total_tweets_grabbed)

                    # Stop with 3 queries left before hitting the limit
                    # just to be safe
                    if ('x-rate-limit-remaining' in self.ts.get_metadata() and
                            int(self.ts.get_metadata()['x-rate-limit-remaining']) <= 3 and
                            now < reset):
                        longsleeptime = reset - now + 60
                        self.log(
                            '\nSleeping %s seconds \n(x-rate-limit-remaining= %s,'
                            ' \nx-rate-limit-reset=%s,'
                            ' \ntime now=%s)'
                            % (
                                longsleeptime,
                                self.ts.get_metadata()[
                                    'x-rate-limit-remaining'],
                                self.ts.get_metadata()['x-rate-limit-reset'],
                                now