Python Stream.disconnect方法代码示例

# 需要导入模块: from tweepy import Stream [as 别名]
# 或者: from tweepy.Stream import disconnect [as 别名]
def main_old():
    pubnub = Pubnub.Pubnub(settings.PUBNUB_PUBLISH_KEY, settings.PUBNUB_SUBSCRIBE_KEY, settings.PUBNUB_SECRET, False)
    hashtags = set()

    auth = OAuthHandler(settings.TWITTER_CONSUMER_KEY, settings.TWITTER_CONSUMER_SECRET)
    auth.set_access_token(settings.TWITTER_ACCESS_TOKEN, settings.TWITTER_ACCESS_TOKEN_SECRET)
    stream = None
    print "Started"
    while True:
        try:
            #current_hashtags = set(w.hashtag for w in Wall.objects.filter(last_ping__gt=datetime.datetime.now(utc) - datetime.timedelta(minutes=settings.WALL_EXPIRATION)))
            current_hashtags = set(w.hashtag for w in Wall.objects.all())
            if len(current_hashtags - hashtags) > 0:
                if stream is not None:
                    stream.disconnect()
                stream = Stream(auth, PubnubListener(pubnub))
                hashtags = current_hashtags
                print("Now filtering " + ", ".join(list(hashtags)))
                stream.filter(track=list(hashtags), async=True)
            time.sleep(5)
        except KeyboardInterrupt:
            stream.disconnect()
            break
        except Exception as e:
            print e
        time.sleep(5)

# 需要导入模块: from tweepy import Stream [as 别名]
# 或者: from tweepy.Stream import disconnect [as 别名]
class StreamConsumerThreadClass(threading.Thread):
    def __init__(self,term='',oauthfile=''):
        threading.Thread.__init__(self)
        self.searchterm = term
        self.name = term
        self.consume = True
        
        oauth = json.loads(open(oauthfile,'r').read())
        
        listener = MongoDBListener()
        auth = OAuthHandler(oauth['consumer_key'], oauth['consumer_secret'])
        auth.set_access_token(oauth['access_token'], oauth['access_token_secret'])
    
        self.stream = Stream(auth, listener,timeout=60)  
        
        
    def stopConsume(self):
        self.stream.disconnect()
      
    def run(self):
        now = datetime.datetime.now()
        print "Twitter Stream with terms: %s started at: %s" % (self.getName(), now)
        
        connected = False
        while True:
            try: 
                if not connected:
                    connected = True
                    self.stream.filter(track=[self.searchterm])
            except SSLError, e:
                print e
                connected = False            

# 需要导入模块: from tweepy import Stream [as 别名]
# 或者: from tweepy.Stream import disconnect [as 别名]
    def KeywordUpdateModule():

        global KeywordsToFilter

        print("Authenticating Keyword Updater")

        auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
        auth.secure = True
        auth.set_access_token(access_token, access_token_secret)
        myApi = tweepy.API(auth)
        myKeywordStream = Stream(auth, tweepy.StreamListener())

        print("Authentication successfull!")

        keywords = list()

        timeline = myApi.home_timeline()
        for raw_data in timeline:
            message = raw_data.text
            kw = indicoKeywords(message)
            for keyword in kw:
                keywords.append(unicode(keyword.decode('utf-8')))
        print("Keywords collected, sleeping...")
        myKeywordStream.disconnect()

        print(keywords)
        KeywordsToFilter = keywords

# 需要导入模块: from tweepy import Stream [as 别名]
# 或者: from tweepy.Stream import disconnect [as 别名]
class TwitterListener(object):

    def __init__(self):
        pubnub = Pubnub.Pubnub(settings.PUBNUB_PUBLISH_KEY, settings.PUBNUB_SUBSCRIBE_KEY, settings.PUBNUB_SECRET, False)

        self.pubnub_listener = PubnubListener(pubnub)
        self.stream = None
        self.hashtags = set()
        self.auth = OAuthHandler(settings.TWITTER_CONSUMER_KEY, settings.TWITTER_CONSUMER_SECRET)
        self.auth.set_access_token(settings.TWITTER_ACCESS_TOKEN, settings.TWITTER_ACCESS_TOKEN_SECRET)

    def update_hashtags(self):
        current_hashtags = set(w.hashtag for w in Wall.objects.all())
        if len(current_hashtags - self.hashtags) > 0:
            self.hashtags = current_hashtags
            return True

    def filter(self):
        if self.stream is not None:
            self.stream.disconnect()
        self.stream = Stream(self.auth, self.pubnub_listener)

        print("Now filtering " + ", ".join(list(self.hashtags)))

        self.stream.filter(track=list(self.hashtags), async=True)

    def update(self):
        if self.update_hashtags():
            self.filter()

    def exit(self):
        if self.stream is not None:
            self.stream.disconnect()

# 需要导入模块: from tweepy import Stream [as 别名]
# 或者: from tweepy.Stream import disconnect [as 别名]
def main():
    while True:
        try:
            with utils.connect('ng') as conn:
                with conn.cursor() as cur:
                    cur.execute("""CREATE TABLE IF NOT EXISTS {tablename}(
                        id_str text PRIMARY KEY,
                        source text,
                        user_id text,
                        created_at timestamp,
                        text text)""".format(tablename=TABLENAME))
                data_streamer = tweet.PostgresStreamer(conn=conn, tablename=TABLENAME)
                auth = OAuthHandler(CONSUMER_KEY, CONSUMER_SECRET)
                auth.set_access_token(ACCESS_TOKEN_KEY, ACCESS_TOKEN_SECRET)
                stream = Stream(auth, data_streamer)
                stream.sample(languages=LANGUAGES)
                conn.commit()
        except KeyboardInterrupt:
            stream.disconnect()
            break
        except (IndexError, ConnectionError, ProtocolError, ReadTimeoutError):
            #logger.exception(e)
            stream.disconnect()
            time.sleep(90)
            continue

# 需要导入模块: from tweepy import Stream [as 别名]
# 或者: from tweepy.Stream import disconnect [as 别名]
class TweetController:
    """docstring for Controller"""

    def __init__(self):
        self.settings = Settings()
        # self.auth = OAuthHandler(Listener.api_data["consumer_key"], Listener.api_data["consumer_secret"])
        # self.auth.set_access_token(Listener.api_data["access_token"], Listener.api_data["access_token_secret"])
        self.api = tweepy.API(self.settings.auth, parser=tweepy.parsers.JSONParser())

        self.db = DataBase()
        # self.tweet_gui = tweet_gui
        self.default_keyword = ['Obama', 'hillary ', 'Trump']
        self.db.create_table_if_not_exist()

    def start_stream(self):
        self.tweet_listener = Listener()
        self.stream = Stream(auth=self.settings.auth, listener=self.tweet_listener)
        self.stream.filter(track=self.default_keyword, async=True)

    def stop_stream(self):
        self.stream.disconnect()

    def set_keyword(self, default_keyword):
        self.default_keyword = default_keyword
        print(default_keyword)

# 需要导入模块: from tweepy import Stream [as 别名]
# 或者: from tweepy.Stream import disconnect [as 别名]
class StreamConsumerThreadClass(threading.Thread):
    def __init__(self, term="", oauthfile="", follow=False, user=False, track=False):
        threading.Thread.__init__(self)
        self.searchterm = term
        self.name = term
        self.consume = True
        self.follow = follow
        self.user = user
        self.track = track
        listener = MongoDBListener()

        try:
            oauth = json.loads(open(oauthfile, "r").read())

            if "consumer_key" in oauth:
                auth = OAuthHandler(oauth["consumer_key"], oauth["consumer_secret"])
                auth.set_access_token(oauth["access_token"], oauth["access_token_secret"])
                self.api = API(auth)

                if not self.api.verify_credentials():
                    raise Exception("Invalid credentials")

                self.stream = Stream(auth, listener, timeout=60)

        except:
            print "Error logging to Twitter"
            raise

    def stop_consume(self):
        self.stream.disconnect()

    def run(self):
        now = datetime.datetime.now()
        if self.user:
            print "Twitter Stream of the OAuth user: started at: %s" % (now)
        else:
            print "Twitter Stream with terms: %s started at: %s" % (self.getName(), now)

        connected = False
        while True:
            try:
                if not connected:
                    connected = True
                    if self.user:
                        self.stream.userstream(_with="followings")
                    elif self.follow:
                        user_ids = []
                        for user in self.api.lookup_users([], self.searchterm.split(","), False):
                            user_ids.append(user.id)
                        self.stream.filter(follow=[",".join("{0}".format(n) for n in user_ids)])
                    elif self.track:
                        self.stream.filter(track=[self.searchterm])

            except SSLError, sse:
                print sse
                connected = False
            except Exception, e:
                print "Stream error"
                raise e

# 需要导入模块: from tweepy import Stream [as 别名]
# 或者: from tweepy.Stream import disconnect [as 别名]
def listenTweets(waitTime=60):
    l = myStreamListener()
    auth = OAuthHandler(CONSUMER_KEY, CONSUMER_SECRET)
    auth.set_access_token(ACCESS_TOKEN, ACCESS_TOKEN_SECRET)
    stream = Stream(auth, l)
    stream.filter(track=BRAND_NAMES, async= True)
    time.sleep(waitTime)
    print "Disconnecting"  
    stream.disconnect()

# 需要导入模块: from tweepy import Stream [as 别名]
# 或者: from tweepy.Stream import disconnect [as 别名]
class TweetDownloader(safethread.SafeThread, StreamListener):

    def __init__(self,
                 destpath,
                 consumer_key,
                 consumer_secret,
                 access_token,
                 access_secret,
                 window = 10000,
                 verbose = False):
        super(TweetDownloader, self).__init__(name="TweetDownloader")
        self.destpath = destpath
        self.consumer_key = consumer_key
        self.consumer_secret = consumer_secret
        self.access_token = access_token
        self.access_secret = access_secret
        self.prefix = 'tweets'
        self.suffix = 'txt'
        self.window = window
        self.buf = collections.deque()
        self.stopped = True

    # Write the tweet text to the current file. May throw an error if the file
    # is currently being switched out (i.e. writing at the end of a window).
    def write(self, vals):
        self.buf.appendleft(json.dumps(vals))

    def action(self):
        if len(self.buf) > 0:
            self.f.write(self.buf.pop() + '\n')

        if ((time.time() * 1000) - self.begin > self.window):
            self.f.close()
            fname = self.destpath + self.prefix + '-' + str(self.begin) + \
                    '.' + self.suffix
            os.rename(self.destpath + 'tmp', fname)

            self.begin = int(time.time() * 1000)
            self.f = open(self.destpath + 'tmp', 'w')

    def start(self):
        # Setup the stream
        auth = OAuthHandler(self.consumer_key, self.consumer_secret)
        auth.set_access_token(self.access_token, self.access_secret)
        self.stream = Stream(auth, TweetListener(self))

        # Create the first file
        self.begin = int(time.time() * 1000)
        self.f = open(self.destpath + 'tmp', 'w')

        # Start the threads
        self.stream.sample(async=True)
        super(TweetDownloader, self).start()

    def stop(self):
        self.stream.disconnect()
        super(TweetDownloader, self).stop()

# 需要导入模块: from tweepy import Stream [as 别名]
# 或者: from tweepy.Stream import disconnect [as 别名]
def main():
    print("Streaming started.... Ctrl+C to abort")
    try:
        twitterStream = Stream(auth,listener())
        twitterStream.filter(locations = GEOBOX_BHAM)
    except Exception as e:
        print (e)
        print("Error or execution finished. Program exiting... ")
        twitterStream.disconnect()

# 需要导入模块: from tweepy import Stream [as 别名]
# 或者: from tweepy.Stream import disconnect [as 别名]
def start_streaming(track=[""], locations=[-180,-90,180,90], languages=["en"]):
    l = StdOutListener()
    stream = Stream(auth, l)
    while True:
        try:
            stream.disconnect()
            stream.filter(track=track, locations=locations, languages=languages)
        except Exception, e:
            print "Exception: ", e

# 需要导入模块: from tweepy import Stream [as 别名]
# 或者: from tweepy.Stream import disconnect [as 别名]
class streamer :


    ##accepts initial hashtag (without #) as parameter
    def __init__(self,) :
        self.tstreamer = None
        self.data = None
        self.inject_data = None

    ##does what the name implies, stream must first be stopped however
    # returns true if successful
    def set_hashtag(self, hashtag) :
        if (not self.tstreamer) :
            self.data = dataset.dataset(hashtag)
            self.inject_data = self.data.get_injection_method()
            return True
        else :
            return False

    ##returns current hashtag that is set
    # returns none if there is none
    def get_hashtag(self) :
        if (self.data) :
            return self.data.hashtag
        else :
            return None

    ##return true if the streamer is currently running, false if otherwise
    def is_running(self) :
        if (self.tstreamer) :
            return True;
        else :
            return False

    ##starts the streamer, returns true if successful, false if otherwise
    # if it is not successful it means it is already running, or else you did not set
    # the hashtag with the set_hashtag method
    def start_streamer(self) :
        if (not self.tstreamer and self.data) :
            auth = OAuthHandler(ckey, csecret)
            auth.set_access_token(atoken, asecret)
            self.tstreamer = Stream(auth, listener(lambda x : self.inject_data((x,)))) #my silly inject data method takes a list not a string . . . 
            self.tstreamer.filter(track=[self.data.hashtag], async=True)
            return True;
        else :
            return False;

    ##stops the streamer, if it returns false it means that it was not running
    # in the first place
    def stop_streamer(self) :
        if (self.tstreamer) :
            self.tstreamer.disconnect()
            self.tstreamer = None
            return True
        else :
            return False

# 需要导入模块: from tweepy import Stream [as 别名]
# 或者: from tweepy.Stream import disconnect [as 别名]
class SparkStreamListener(StreamListener):
    """ Use twitter streaming API to stream to PySpark. """
    def __init__(self):
        config = ConfigParser()
        config.read(os.path.join(CONFIG_DIR, 'prod.cfg'))
        self.sockets = []
        auth = OAuthHandler(config.get('twitter', 'consumer_key'),
                            config.get('twitter', 'consumer_secret'))
        auth.set_access_token(config.get('twitter', 'access_token'),
                              config.get('twitter', 'access_token_secret'))
        self.stream = Stream(auth, self)

    def add_socket(self, ws):
        self.sockets.append(ws)
        print(self.sockets)

    def run(self):
        try:
            self.stream.filter(track=['python'])
        except Exception as e:
            print(e)
            self.stream.disconnect()

    def start(self):
        """ Start GEvent """
        gevent.spawn(self.run)

    def send(self, status):
        """ Send status to socket """
        print(self.sockets)
        if len(self.sockets) > 1:
            ws = choice(self.sockets)
        else:
            ws = self.sockets[0]
        try:
            ws.send(status.encode('utf-8'))
        except ValueError:
            print(e)
            # the web socket die..
            self.sockets.remove(ws)

    def on_data(self, data):
        decoded = json.loads(data)
        gevent.spawn(self.send, decoded.get('text') + '\n')
        return True

    def on_error(self, status):
        print("Error: %s", status)

    def on_timeout(self):
        print("tweepy timeout.. wait 30 seconds")
        gevent.sleep(30)

# 需要导入模块: from tweepy import Stream [as 别名]
# 或者: from tweepy.Stream import disconnect [as 别名]
def starthashlistener():
    global CONSUMER_KEY,CONSUMER_SECRET,ACCESS_KEY,ACCESS_SECRET
    a = MyHashListener()
    auth = OAuthHandler(CONSUMER_KEY, CONSUMER_SECRET)
    auth.set_access_token(ACCESS_KEY, ACCESS_SECRET)
    stream = Stream(auth, a)
    try:
        print ">>>>> start catching hash <<<<<"
        stream.filter(follow=['4091042235'])
        api.update_status(status = Message)
    except:
        print ">>>>> disconect this stream <<<<<"
        stream.disconnect()

# 需要导入模块: from tweepy import Stream [as 别名]
# 或者: from tweepy.Stream import disconnect [as 别名]
def startlistneruser():
    global CONSUMER_KEY,CONSUMER_SECRET,ACCESS_KEY,ACCESS_SECRET
    l = MyMessageListener()
    auth = OAuthHandler(CONSUMER_KEY, CONSUMER_SECRET)
    auth.set_access_token(ACCESS_KEY, ACCESS_SECRET)
    stream = Stream(auth, l)
    try:
        ">>>>> start listening to stream <<<<<"
        stream.filter(track=['yolo'])
        api.update_status(status = Message)
    except:
        print ">>>>> disconect this stream <<<<<"
        stream.disconnect()