Python Stream.sample方法代码示例

# 需要导入模块: from tweepy import Stream [as 别名]
# 或者: from tweepy.Stream import sample [as 别名]
def interactive(username=None, password=None, filenames=None):
    if not username:
        username = raw_input('Username: ').strip()
    if not password:
        password = getpass("Password: ").strip()
    s = Stream(username, password, TWSSBuildClassifierListner())
    s.sample()

# 需要导入模块: from tweepy import Stream [as 别名]
# 或者: from tweepy.Stream import sample [as 别名]
def main():
    while True:
        try:
            with utils.connect('ng') as conn:
                with conn.cursor() as cur:
                    cur.execute("""CREATE TABLE IF NOT EXISTS {tablename}(
                        id_str text PRIMARY KEY,
                        source text,
                        user_id text,
                        created_at timestamp,
                        text text)""".format(tablename=TABLENAME))
                data_streamer = tweet.PostgresStreamer(conn=conn, tablename=TABLENAME)
                auth = OAuthHandler(CONSUMER_KEY, CONSUMER_SECRET)
                auth.set_access_token(ACCESS_TOKEN_KEY, ACCESS_TOKEN_SECRET)
                stream = Stream(auth, data_streamer)
                stream.sample(languages=LANGUAGES)
                conn.commit()
        except KeyboardInterrupt:
            stream.disconnect()
            break
        except (IndexError, ConnectionError, ProtocolError, ReadTimeoutError):
            #logger.exception(e)
            stream.disconnect()
            time.sleep(90)
            continue

# 需要导入模块: from tweepy import Stream [as 别名]
# 或者: from tweepy.Stream import sample [as 别名]
def printStream():
    #This handles Twitter authetification and the connection to Twitter Streaming API
    l = StdOutListener()
    auth = OAuthHandler(CONSUMER_KEY, CONSUMER_SECRET)
    auth.set_access_token(ACCESS_KEY, ACCESS_SECRET)
    stream = Stream(auth, l)
    stream.sample()

# 需要导入模块: from tweepy import Stream [as 别名]
# 或者: from tweepy.Stream import sample [as 别名]
def run_sample():
    l = TweetListener()
    auth = OAuthHandler(consumer_key, consumer_secret)
    auth.set_access_token(access_token, access_token_secret)

    stream = Stream(auth, l)
    stream.sample()

# 需要导入模块: from tweepy import Stream [as 别名]
# 或者: from tweepy.Stream import sample [as 别名]
class streamsample(StreamListener): 
   
    # constructor for stream
    def __init__(self):
       l = StdOutListener() 
       auth = OAuthHandler(consumer_key, consumer_secret)
       auth.set_access_token(access_token, access_token_secret) 
       self.stream = Stream(auth, l)    

    def printtest(self):
	self.stream.sample()
	
    # Sends tweet to the sqlite3 database
    def todatabase(self, numtweets = 100):
	
	connection = sqlite3.connect(DATABASE_NAME)
	cursor = connection.cursor()

	try:
	    count = 0
	    while count < numtweets:
		for tweet_json in self.stream:
		    #check whether the tweet is actually a tweet
		    if tweet.get('text') and tweet['user']['lang'] == 'en':
			parsetweet(tweet_json)
		    break
		count += 1

	except stream.ConnectionError, e:
	    print "Disconnected"

# 需要导入模块: from tweepy import Stream [as 别名]
# 或者: from tweepy.Stream import sample [as 别名]
def start_stream(auth, l):
    while True:
        try:
            stream = Stream(auth, l)
            stream.sample()
        except:
            continue

# 需要导入模块: from tweepy import Stream [as 别名]
# 或者: from tweepy.Stream import sample [as 别名]
    def get_streaming_data(self):
        tweets_grabbed = 0
        while (tweets_grabbed < self.num_tweets_to_grab):
            twitterStream = Stream(self.auth, listener(self.s, self.twit_utils, self.num_tweets_to_grab, self.retweet_count))
            try:               
                twitterStream.sample()
            except Exception as e:
                    print("Error. Restarting Stream.... Error: ")
                    print(e.__doc__)
                    #print(e.message)
                    print("Le Error! Restart")
                    time.sleep(3) # Sleep for 5 minutes if error ocurred
            finally:
                tweets_grabbed = self.s.get_tweets_grabbed()
                print("tweets_grabbed = ", tweets_grabbed)

        lang, top_lang,love_words, swear_words, top_tweets, countries = self.s.get_stats()

        print(Counter(lang))
        print(Counter(top_lang))
        print("Love Words {} Swear Words {}".format(love_words, swear_words))
        print(Counter(countries))

        self.c.execute("INSERT INTO lang_data VALUES (?,?, DATETIME('now'))", (str(list(Counter(lang).items())), str(list(Counter(top_lang).items()))))

        self.c.execute("INSERT INTO love_data VALUES (?,?, DATETIME('now'))", (love_words, swear_words))

        for t in top_tweets:
            self.c.execute("INSERT INTO twit_data VALUES (?, DATETIME('now'))", (t,))

        self.c.execute("INSERT INTO country_data VALUES (?, DATETIME('now'))", (str(list(Counter(countries).items())),))

        self.conn.commit()

# 需要导入模块: from tweepy import Stream [as 别名]
# 或者: from tweepy.Stream import sample [as 别名]
def sample():
    '''Get random sample of all public statuses.'''
    listener = TweetStreamListener()
    auth = OAuthHandler(consumer_key, consumer_secret)
    auth.set_access_token(access_token, access_secret)

    stream = Stream(auth, listener)
    stream.sample()

# 需要导入模块: from tweepy import Stream [as 别名]
# 或者: from tweepy.Stream import sample [as 别名]
 def get_streaming_data(self):
     twitter_stream = Stream(
         self.auth, twitter_listener(num_tweets_to_grab=self.num_tweets_to_grab, retweet_count=self.retweet_count)
     )
     try:
         twitter_stream.sample()
     except Exception as e:
         print(e.__doc__)

# 需要导入模块: from tweepy import Stream [as 别名]
# 或者: from tweepy.Stream import sample [as 别名]
class TweetDownloader(safethread.SafeThread, StreamListener):

    def __init__(self,
                 destpath,
                 consumer_key,
                 consumer_secret,
                 access_token,
                 access_secret,
                 window = 10000,
                 verbose = False):
        super(TweetDownloader, self).__init__(name="TweetDownloader")
        self.destpath = destpath
        self.consumer_key = consumer_key
        self.consumer_secret = consumer_secret
        self.access_token = access_token
        self.access_secret = access_secret
        self.prefix = 'tweets'
        self.suffix = 'txt'
        self.window = window
        self.buf = collections.deque()
        self.stopped = True

    # Write the tweet text to the current file. May throw an error if the file
    # is currently being switched out (i.e. writing at the end of a window).
    def write(self, vals):
        self.buf.appendleft(json.dumps(vals))

    def action(self):
        if len(self.buf) > 0:
            self.f.write(self.buf.pop() + '\n')

        if ((time.time() * 1000) - self.begin > self.window):
            self.f.close()
            fname = self.destpath + self.prefix + '-' + str(self.begin) + \
                    '.' + self.suffix
            os.rename(self.destpath + 'tmp', fname)

            self.begin = int(time.time() * 1000)
            self.f = open(self.destpath + 'tmp', 'w')

    def start(self):
        # Setup the stream
        auth = OAuthHandler(self.consumer_key, self.consumer_secret)
        auth.set_access_token(self.access_token, self.access_secret)
        self.stream = Stream(auth, TweetListener(self))

        # Create the first file
        self.begin = int(time.time() * 1000)
        self.f = open(self.destpath + 'tmp', 'w')

        # Start the threads
        self.stream.sample(async=True)
        super(TweetDownloader, self).start()

    def stop(self):
        self.stream.disconnect()
        super(TweetDownloader, self).stop()

# 需要导入模块: from tweepy import Stream [as 别名]
# 或者: from tweepy.Stream import sample [as 别名]
 def _get_tweets(self):
     auth = OAuthHandler(
         self.conf['oauth.consumer_key'],
         self.conf['oauth.consumer_secret'])
     auth.set_access_token(
         self.conf['oauth.access_token'],
         self.conf['oauth.access_token_secret'])
     stream = Stream(auth, QueueListener(self.queue))
     stream.sample(languages=['en'])

# 需要导入模块: from tweepy import Stream [as 别名]
# 或者: from tweepy.Stream import sample [as 别名]
    def get_streaming_data(self):
        twitter_stream = Stream(self.auth, twitter_listener(num_tweets_to_grab=self.num_tweets_to_grab, retweet_count = self.retweet_count, stats = self.stats, get_tweet_html = self.get_tweet_html ))
        try:
            twitter_stream.sample()
        except Exception as e:
            print(e.__doc__)

        lang, top_lang, top_tweets = self.stats.get_stats()
        print(Counter(lang))
        print(Counter(top_lang))
        print(top_tweets)

# 需要导入模块: from tweepy import Stream [as 别名]
# 或者: from tweepy.Stream import sample [as 别名]
 def start_streaming(self):
     # start streaming
     while True:
         try:
             stream = Stream(self.auth_handler, self)
             print ('Listening...')
             stream.sample(languages=['en'])
         except Exception as e:
             # reconnect on exceptions
             print (e)
             continue

# 需要导入模块: from tweepy import Stream [as 别名]
# 或者: from tweepy.Stream import sample [as 别名]
class TwitterProducer(Producer):
    TOPIC = "twitter"

    def __init__(self, *args, **kwargs):
        Producer.__init__(self, *args, **kwargs)
        self.auth = OAuthHandler(twitter.get("ckey"), twitter.get("csecret"))
        self.auth.set_access_token(twitter.get("atoken"), twitter.get("asecret"))
        self.li = TwitterListener(self.queue)
        self.stream = Stream(self.auth, self.li)

    def produce(self):
        self.stream.sample(languages=["en"])

# 需要导入模块: from tweepy import Stream [as 别名]
# 或者: from tweepy.Stream import sample [as 别名]
    def get_random_sample(self, size):
        """
        Get a random sample of tweets for T_n (10x size of T_b): We need a json? Sampling is
        """
        auth = OAuthHandler(CONSUMER_KEY, CONSUMER_SECRET)
        auth.set_access_token(ACCESS_TOKEN, ACCESS_TOKEN_SECRET)
        l = SampleStreamListener(size)
        stream = Stream(auth, l)
        stream.sample()
        tweets = l.get_sample()

        return tweets