Python Httpy.Httpy类代码示例

	def test():
		'''
			Test that ripper is working as expected.
			Raise exception if necessary.
		'''
		from Httpy import Httpy
		httpy = Httpy()

		# Check we can hit the host
		url = 'http://imgur.com'
		r = httpy.get(url)
		if len(r.strip()) == 0:
			raise Exception('unable to retrieve data from %s' % url)

		# Check ripper gets all images in an album
		#url = 'http://markedone911.imgur.com/'
		#url = 'http://imgur.com/r/nsfw_oc/top/all'
		url = SiteImgur.get_sample_url()
		s = SiteImgur(url)
		urls = s.get_urls()
		for (i,u) in enumerate(urls):
			print i, u
		expected = 4
		if len(urls) < expected:
			return 'expected at least %d images, got %d. url: %s' % (expected, len(urls), url)
		return None

	def get_urls_user_albums(self):
		if self.url.endswith('/all'):
			# Images, not albums
			return self.get_urls_user_images()

		from Httpy import Httpy
		httpy = Httpy()

		user = self.url.split('//')[1].split('.')[0]
		r = httpy.get(self.url)
		result = []
		for (index, cover) in enumerate(httpy.between(r, '<div class="cover">', '</div>')):
			if not '<a href="' in cover: continue
			album = httpy.between(cover, '<a href="', '"')[0]
			if album.startswith('//'):
				album = 'http:%s' % album
			albumid = album.split('/')[4]
			album = 'http://imgur.com/a/%s' % albumid
			for image in self.get_urls_album(album):
				# Tack this album's index/albumid to image
				image['saveas'] = '%03d_%s_%s' % (index + 1, albumid, image['saveas'])
				result.append(image)
			sleep(2)
			if len(result) > SiteBase.MAX_IMAGES_PER_RIP:
				break
		return result

	def test():
		'''
			Test that ripper is working as expected.
			StatusManager.py uses the results of this method to show what rippers are working/broken on the main page

			Returns:
				None - if ripper is working as expected
				str - Warning message if the ripper may not be working properly.

			Raises:
				Exception - if ripper is definitely broken. Exception message is used to display on site.
		'''
		from Httpy import Httpy
		httpy = Httpy()

		# Check we can hit the host
		url = 'http://hostname.com'
		r = httpy.get(url)
		if len(r.strip()) == 0:
			# Raise exception because the site is *very* broken, definitely can't rip from it if we can't hit the home page.
			raise Exception('unable to retrieve data from %s' % url)

		# Check ripper gets all images in an album
		url = _SampleSite.get_sample_url()
		s = _SampleSite(url)
		urls = s.get_urls()
		expected = 10
		if len(urls) < expected:
			# Returning non-None string since this may be a transient error.
			# Maybe the album was deleted but the ripper is working as expected.
			return 'expected at least %d images, got %d. url: %s' % (expected, len(urls), url)

		# Returning None because the ripper is working as expected. No issues found.
		return None

	def get_urls(self):
		self.api_key = self.db.get_config('tumblr_key')
		if self.api_key == None:
			raise Exception('unable to rip album (%s), tumblr key not found in database' % self.url)

		from Httpy import Httpy
		httpy = Httpy()

		result = []
		offset = 0
		while True:
			url = self.get_api_url(offset=offset)
			r = httpy.get(url)
			json = loads(r)
			if not 'response' in json or not 'posts' in json['response']:
				#raise Exception('no posts found at %s' % self.url)
				break

			posts = json['response']['posts']
			if len(posts) == 0: break

			for post in posts:
				for photos in post['photos']:
					result.append(photos['original_size']['url'])
			if self.post_type == 'post': break
			if len(result) > SiteBase.MAX_IMAGES_PER_RIP:
				break
			offset += 20
			sleep(1)
		return result

	def get_image_count_for_album(url):
		url = url.replace('m.imgur.com', 'imgur.com').replace('https://', '').replace('http://', '')
		aid = url.split('/')[2]
		url = 'http://imgur.com/a/%s/noscript' % aid
		httpy = Httpy()
		r = httpy.get(url)
		return r.count('src="//i.imgur.com')

	def test():
		from Httpy import Httpy
		httpy = Httpy()
		try:
			r = httpy.get('http://www.vimeo.com/')
			if len(r.strip()) == 0:
				raise Exception('empty response from vimeo.com')
		except Exception, e:
			raise e

	def get_urls(self):
		from Httpy import Httpy
		httpy = Httpy()

		r = httpy.get(self.url)
		result = []
		for link in httpy.between(r, '/img.php?path=', '"'):
			result.append(link)
		return result

	def sanitize_url(self):
		if '/image.php?id=' in self.url:
			from Httpy import Httpy
			httpy = Httpy()
			r = httpy.get(self.url)
			if not 'View complete gallery: <a href="' in r:
				raise Exception('no gallery found at %s' % self.url)
			self.url = 'http://imagearn.com/%s' % httpy.between(r, 'View complete gallery: <a href="', '"')[0]
		if not '/gallery.php?id=' in self.url:
			raise Exception('expected /gallery.php?id= not found in URL')

	def get_urls(self):
		from Httpy import Httpy
		httpy = Httpy()

		r = httpy.get(self.url)
		result = []
		for post in httpy.between(r, 'daposts">', '</div> </div> </div>'):
			images = httpy.between(post, 'href="', '"')
			if len(images) > 0 and 'javascript:' not in images[0]:
				result.append('http://www.chansluts.com%s' % images[0])
		return result

	def sanitize_url(self):
		if '/image/' in self.url:
			from Httpy import Httpy
			httpy = Httpy()
			r = httpy.get(self.url)
			if not "class='gallery_title'><a href='" in r:
				raise Exception('no gallery found at %s' % self.url)
			self.url = httpy.between(r, "class='gallery_title'><a href='", "'")[0]
		if not '/gallery/' in self.url:
			raise Exception('expected /gallery/ not found in URL')
		if not self.url.endswith('/'): self.url += '/'

	def get_urls(self):
		from Httpy import Httpy
		httpy = Httpy()

		fields = self.url.split('/')
		url = 'http://api.4chan.org/%s/res/%s.json' % (fields[3], fields[5])
		try:
			r = httpy.get(url)
			json = loads(r)
			posts = json['posts']
		except Exception, e:
			raise Exception('failed to load %s: %s' % (url, str(e)))

	def get_urls(self):
		from Httpy import Httpy
		httpy = Httpy()
		r = httpy.get(self.url)
		r = r[r.find('showMoreGalleries'):] # To ignore user icon
		links = httpy.between(r, 'border=0 src="', '"')
		result = []
		for link in links:
			link = 'http://%s' % link[link.find('.')+1:].replace('/images/thumb/', '/images/full/')
			result.append(link)
			if len(result) > SiteBase.MAX_IMAGES_PER_RIP:
				break
		return result

	def get_urls(self):
		'''
			Returns list of URLs from album. Does not download them.
		'''
		from Httpy import Httpy
		httpy = Httpy()

		r = httpy.get(self.url)
		result = []
		for link in httpy.between(r, '<img src="', '"'):
			link = 'http://hostname.com%s' % link
			result.append(link)
			if len(result) > SiteBase.MAX_IMAGES_PER_RIP:
				break
		return result

	def searchImages(unfiltered_search_text, start_index, source_ip='127.0.0.1', safe='off'):
		search_text = unfiltered_search_text.replace(' ', '%20')

		url = 'https://ajax.googleapis.com/ajax/services/search/images?v=1.0'
		url +=      '&q=%s' % search_text.replace(' ', '%20')
		url +=  '&start=%d' % start_index
		url += '&userip=%s' % source_ip
		url +=   '&safe=%s' % safe

		from Httpy import Httpy
		httpy = Httpy()
		try:
			response = httpy.get(url)
		except Exception, e:
			raise e

	def get_urls(self):
		from Httpy import Httpy
		httpy = Httpy()

		r = httpy.get(self.url)
		result = []
		for link in httpy.between(r, 'src="', '"'):
			if not 'http://' in link: continue
			if not 'imgur.com' in link: continue
			doti = link.rfind('.')-1
			if link[doti] == 'm':
				link = link.replace(link[doti:], link[doti+1:])
			result.append(link)
			if len(result) > SiteBase.MAX_IMAGES_PER_RIP:
				break
		return result