Python Twython.search_gen方法代码示例

# 需要导入模块: from twython import Twython [as 别名]
# 或者: from twython.Twython import search_gen [as 别名]
def search_for(str):
	oauth = get_oauthentication()
	twitter = Twython(oauth['APP_KEY'], oauth['APP_SECRET'], oauth_version=2)
	ACCESS_TOKEN = twitter.obtain_access_token()
	twitter = Twython(oauth['APP_KEY'], access_token=ACCESS_TOKEN)
	
	search = twitter.search_gen(str)
	# search = twitter.get_home_timeline()
			
	for result in search:
		print result['text']

# 需要导入模块: from twython import Twython [as 别名]
# 或者: from twython.Twython import search_gen [as 别名]
class twython_api:
    def __init__(self):
        self.TWITTER = Twython(settings.APP_KEY, access_token=settings.ACCESS_TOKEN)

    def search(self, word, type=None, max=None):
        # Only search
        if type == None and max == None:
            data = self.TWITTER.search(q=word)
            return data

        # Search with Recent Type
        elif max == None:
            # result_type:
            # Specifies what type of search results you would prefer to receive.
            # The current default is 'mixed'.
            # Valid values include:
                # mixed: Include both popular and real time results in the response.
                # popular: Return only the most recent results in the response.
                # popular: Return only the most popular results in the response.
            data = self.TWITTER.search(q=word, result_type=type)
            return data

        # Search with Recent Type and Count
        else:
            # count:
            # The number of tweets to return per page, up to a maximum of 100.
            # Defaults to 15.
            # This was formerly the 'rpp' parameter in the old Search API.
            data = self.TWITTER.search(q=word, result_type=type, count=max)
            return data

    # Returns Generator
    def search_gen(self, word):
        # This function offers a generator for search results
        data = self.TWITTER.search_gen(word)
        return data

        # Use of search_gen
        # twitter = twython_api()
        # data = twitter.search_gen('#bitcoin')
        # for result in data:
        #     print(result)

    # Don't use for search_gen. Only use for search.
    def get_tweet_list_by_html(self, data):
        tweet_list = []
        for result in data['statuses']:
            result['text'] = Twython.html_for_tweet(result, use_display_url=True, use_expanded_url=True)
            tweet_list.append(result['text'])
        return tweet_list

# 需要导入模块: from twython import Twython [as 别名]
# 或者: from twython.Twython import search_gen [as 别名]
def fetch_tweets(search_request):
    """
    fetches tweets from Twitter API extracts urls and updates db
    """
    twitter = Twython(
        settings.TWITTER_CONSUMER_KEY,
        settings.TWITTER_CONSUMER_SECRET,
        settings.TWITTER_ACCESS_TOKEN,
        settings.TWITTER_ACCESS_TOKEN_SECRET,
    )

    tweets = twitter.search_gen(search_request)
    found_urls = extract_urls(tweets)

    search_keyword_object = SearchKeyWord()
    search_keyword_object.gifs = found_urls
    search_keyword_object.search_keyword = search_request
    search_keyword_object.updated_at = datetime.now()
    print (search_keyword_object)
    search_keyword_object.save()
    return found_urls

# 需要导入模块: from twython import Twython [as 别名]
# 或者: from twython.Twython import search_gen [as 别名]
class Crawler(object):
    def __init__(self, noTwi=False):
        self.conn = Connection()
        self.db = self.conn.twit
        self.app_key = 'gnkaxR5nCIkkj64P0Vdlg'
        self.app_secret = 'GoxnQjhFEmLgEBwEK0jdQuyY8txAkeX1ma4eIwqBbc'
        if not noTwi:
            self.twi_auth()

    def twi_auth(self):
        """docstring for twi_auth"""
        twitter = Twython(self.app_key, self.app_secret, oauth_version=2)
        self.access_token = twitter.obtain_access_token()
        self.twitter = Twython(self.app_key, access_token=self.access_token)

    def to_datetime(self, string):
        #time_format = '%a, %d %b %Y %H:%M:%S +0000'
        time_format = '%a %b %d %H:%M:%S +0000 %Y'
        # ValueError: time data 'Wed Sep 04 20:53:47 +0000 2013' does not match
        # format '%a, %d %b %Y %H:%M:%S +0000'

        t = time.strptime(string, time_format)
        return datetime(
            year=t.tm_year,
            month=t.tm_mon,
            day=t.tm_mday,
            hour=t.tm_hour,
            minute=t.tm_min,
            second=t.tm_sec)

    def fetch_tweets(self, htags):
        assert self.twitter is not None, 'Twitter API is disabled!'

        def get_tweets_from_api(htag):
            #print(htag)
            r = [_ for _ in self.twitter.search_gen(htag)]
            #print(r)
            #print()
            #print(res)
            #print('{tag} fetched...'.format(tag=htag))
            return r

        res = {}
        #print('scaning htags...')
        rate_limit = {}
        for htag in htags:
            rate_limit = self.twitter\
                .get_application_rate_limit_status(resources='search')[
                    'resources']['search']['/search/tweets']
            #print(rate_limit)
            if rate_limit['remaining'] < 1 and \
                    rate_limit['reset'] > time.time():
                time.sleep(rate_limit['reset'] - time.time())

            try:
                res[htag] = get_tweets_from_api(htag)
            except TwythonRateLimitError as e:
                rate_limit = self.twitter\
                    .get_application_rate_limit_status(resources='search')[
                        'resources']['search']['/search/tweets']
                #print('Rate limit: {0}'.format(rate_limit))
                #print(e)
                if rate_limit['reset'] > time.time():
                    time.sleep(rate_limit['reset'] - time.time())
                res[htag] = get_tweets_from_api(htag)
            except e:
                #print(e)
                pass
        return res

    def check_tweet(self, tweet):
        db_tweet = self.db.tweet.find_one({'id': tweet['id']})
        if db_tweet:
            return True
        else:
            return False

    def save_tweet(self, htag, tweet):
        tweet_data = {
            'htags': [htag] + [
                '#{0}'.format(ht['text'])
                for ht in tweet['entities']['hashtags']
            ],
            'id': tweet['id'],
            'datetime': self.to_datetime(tweet['created_at']),
            'from_user': tweet['user']['screen_name'],
            'profile_image_url': tweet['user']['profile_image_url'],
            'text': tweet['text'],
            'raw': tweet
        }
        db_tweet = self.check_tweet(tweet)
        #print(db_tweet)
        if not db_tweet:
            try:
                self.db.tweet.insert(tweet_data)
                #print('Tweet was recorded')
                for ht in tweet_data['htags']:
                    self.db.meta.insert({
                        'hashtag': ht
                    })
#.........这里部分代码省略.........

# 需要导入模块: from twython import Twython [as 别名]
# 或者: from twython.Twython import search_gen [as 别名]
class TwythonAPITestCase(unittest.TestCase):
    def setUp(self):

        client_args = {
            'headers': {
                'User-Agent': '__twython__ Test'
            },
            'allow_redirects': False
        }

        oauth2_client_args = {
            'headers': {}  # This is so we can hit coverage that Twython sets User-Agent for us if none is supplied
        }

        self.api = Twython(app_key, app_secret,
                           oauth_token, oauth_token_secret,
                           client_args=client_args)

        self.oauth2_api = Twython(app_key, access_token=access_token,
                                  client_args=oauth2_client_args)

    def test_construct_api_url(self):
        """Test constructing a Twitter API url works as we expect"""
        url = 'https://api.twitter.com/1.1/search/tweets.json'
        constructed_url = self.api.construct_api_url(url, q='#twitter')
        self.assertEqual(constructed_url, 'https://api.twitter.com/1.1/search/tweets.json?q=%23twitter')

    def test_get(self):
        """Test Twython generic GET request works"""
        self.api.get('account/verify_credentials')

    def test_post(self):
        """Test Twython generic POST request works, with a full url and
        with just an endpoint"""
        update_url = 'https://api.twitter.com/1.1/statuses/update.json'
        status = self.api.post(update_url, params={'status': 'I love Twython!'})
        self.api.post('statuses/destroy/%s' % status['id_str'])

    def test_get_lastfunction_header(self):
        """Test getting last specific header of the last API call works"""
        self.api.get('statuses/home_timeline')
        self.api.get_lastfunction_header('x-rate-limit-remaining')

    def test_get_lastfunction_header_not_present(self):
        """Test getting specific header that does not exist from the last call returns None"""
        self.api.get('statuses/home_timeline')
        header = self.api.get_lastfunction_header('does-not-exist')
        self.assertEqual(header, None)

    def test_get_lastfunction_header_no_last_api_call(self):
        """Test attempting to get a header when no API call was made raises a TwythonError"""
        self.assertRaises(TwythonError, self.api.get_lastfunction_header,
                          'no-api-call-was-made')

    def test_search_gen(self):
        """Test looping through the generator results works, at least once that is"""
        search = self.api.search_gen('twitter', count=1)
        counter = 0
        while counter < 2:
            counter += 1
            result = next(search)
            new_id_str = int(result['id_str'])
            if counter == 1:
                prev_id_str = new_id_str
                time.sleep(1)  # Give time for another tweet to come into search
            if counter == 2:
                self.assertTrue(new_id_str > prev_id_str)

    def test_encode(self):
        """Test encoding UTF-8 works"""
        self.api.encode('Twython is awesome!')

    def test_html_for_tweet(self):
        """Test HTML for Tweet returns what we want"""
        tweet_text = self.api.html_for_tweet(test_tweet_object)
        self.assertEqual(test_tweet_html, tweet_text)

    def test_html_for_tweet_expanded_url(self):
        """Test using expanded url in HTML for Tweet displays full urls"""
        tweet_text = self.api.html_for_tweet(test_tweet_object,
                                             use_expanded_url=True)
        # Make sure full url is in HTML
        self.assertTrue('http://google.com' in tweet_text)

    def test_html_for_tweet_short_url(self):
        """Test using expanded url in HTML for Tweet displays full urls"""
        tweet_text = self.api.html_for_tweet(test_tweet_object, False)
        # Make sure HTML doesn't contain the display OR exapanded url
        self.assertTrue(not 'http://google.com' in tweet_text)
        self.assertTrue(not 'google.com' in tweet_text)

    def test_raise_error_on_bad_ssl_cert(self):
        """Test TwythonError is raised by a RequestException when an actual HTTP happens"""
        self.assertRaises(TwythonError, self.api.get, 'https://example.com')

    # Timelines
    def test_get_mentions_timeline(self):
        """Test returning mentions timeline for authenticated user succeeds"""
        self.api.get_mentions_timeline()

#.........这里部分代码省略.........