Python utility.read_url函数代码示例

    def trig_mc(self, bot, source, target, trigger, argument):
        term = argument.strip()

        if not term:
            return "usage: .metacritic <game title> or <game> <platform> (slower)"

        url = "http://apps.metacritic.com/search/process?ty=3&tfs=game_title&ts=" + utility.escape(term)
        data = utility.read_url(url)["data"]
        result = self.parse_result(data, term, url)

        if result:
            return result

        error_handler.output_message("<metacritic> title search failed.")
        url = "http://apps.metacritic.com/search/process?ty=3&ts=" + utility.escape(term)
        data = utility.read_url(url)["data"]
        result = self.parse_result(data, term, url)

        if result:
            return result

        return (
            "Found nothing. Try it yourself: "
            + "http://apps.metacritic.com/search/process?ty=3&ts="
            + utility.escape(term)
        )

def prisjakt_search(query_string):
	# Build URLs
	url_product = 'http://www.prisjakt.nu/ajax/jsonajaxserver.php?' + \
			'm=super_search&p={"mode"%3A"' + 'prod_pj' + \
			'"%2C"search"%3A"' + query_string + '"%2C"limit"%3A1%2C"v4"%3A1}'
	url_book = 'http://www.prisjakt.nu/ajax/jsonajaxserver.php?' + \
			'm=super_search&p={"mode"%3A"' + 'bok' + \
			'"%2C"search"%3A"' + query_string + '"%2C"limit"%3A1%2C"v4"%3A1}'

	# Fetch the product result page
	response = utility.read_url(url_product)
	data = response["data"]

	# Look for info
	id_pattern = "\\{'mode': 'produkt', 'produkt_id': '(\\d+)'\\}"
	id_match = re.search(id_pattern, data)

	if not id_match:
		# Fetch the book result page
		response = utility.read_url(url_book)
		data = response["data"]

		# Look for info
		id_pattern = "\\{'mode': 'bok', 'produkt_id': '(\\d+)'\\}"
		id_match = re.search(id_pattern, data)

		url_type = "bok"
	else:
		url_type = "produkt"

	if id_match:
		# We seem to have found something
		product_id = id_match.group(1)

		# Get title
		if url_type == "bok":
			title_pattern = "class=\\\\\"ikon(14)?\\\\\"( alt=\\\\\"\\\\\")?\\> (.+?) \\\\n"
			encoded_title = re.search(title_pattern, data).group(3)
		else:
			title_pattern = "onmouseout=\\\\\"ajaxpopup_hide\(\);\\\\\"\>\\\\n  (.+?) \\\\n"
			encoded_title = re.search(title_pattern, data).group(1)
		
		# Remove HTML tags
		encoded_title = string.replace(
				encoded_title, '<span class=\\"search_hit\\">', '')
		encoded_title = string.replace(encoded_title, '<\\/span>', '')
		# Decode special characters
		product_title = decode_characters(encoded_title)

		# Get price
		data = data.replace("&nbsp;", "")
		price_pattern = "\\<span class=\\\\\"pris\\\\\"\>(\\d+:-)\\<\\\\\/span\\>"
		product_price = re.search(price_pattern, data).group(1)

		# Done, return info string (latin-1 to make IRCClient.send() happy)
		return product_title.encode('latin-1', 'replace') + ", " + product_price + \
				", http://www.prisjakt.nu/" + url_type + ".php?p=" + product_id + \
				" | All results: http://www.prisjakt.nu/search.php?query=" + query_string
	else:
		return "No product found."

def tw_get_info(): 
	counter = 0
	address = "master.teewars.com" 
	master_port = 8300
 
	sock = socket(AF_INET, SOCK_DGRAM) 
	sock.settimeout(5.0) 
	sock.sendto("\x20\x00\x00\x00\x00\x00\xff\xff\xff\xffreqt", (address, master_port)) 
 
	try:
		data, addr = sock.recvfrom(1024) 
		sock.close() 
		data = data[14:] 
		num_servers = len(data) / 6 
		num_players = 0 

		players_dic = {}

		for n in range(0, num_servers): 
			ip = ".".join(map(str, map(ord, data[n*6:n*6+4]))) 
			port = ord(data[n*6+5]) * 256 + ord(data[n*6+4]) 

			#print ip, port

			with list_lock:
				id = thread.start_new_thread(tw_get_num_players_proxy, (ip, port, players_dic))
				players_dic[id] = -2

		while True:
			has_items = False
			with list_lock:
				for slot in players_dic.keys():
					if players_dic[slot] == -2:
						has_items = True
						break

			if has_items:
				time.sleep(0.5)
			else:
				break

		players_list = []

		for slot in players_dic.keys():
			if players_dic[slot] != -1:
				players_list.append(players_dic[slot])

		num_servers = len(players_list)
		num_players = reduce(lambda x, y: x + y, players_list)

		with open("data/tw_stats.txt", "a") as file:
			file.write("%s %s %s\n" % (int(time.time()), num_servers, num_players))
			 
		utility.read_url("http://serp.starkast.net/berserker/gief_stats.php?timestamp=%s&servers=%s&players=%s" % (int(time.time()), num_servers, num_players));
		return (num_servers, num_players)
	except:
		print 'exception O.o', sys.exc_info(), traceback.extract_tb(sys.exc_info()[2])
		return None

 def fetch_show_info(self, show):
     info = {}
     query_result = utility.read_url(self.URL_API % show)
     raw_data = utility.unescape(query_result['data'].decode('utf-8'), True)
     for m in self.PATTERN_DATA_ENTRY.finditer(raw_data):
         info[m.group('key')] = m.group('value').replace('^', u", ")
     return info

def get_title(url):
	import urllib
	if not re.search('[^:]+:\/\/', url):
		url = 'http://' + url

	response = utility.read_url(url)
	if response == None:
		return None

	# workaround ISO-8859-1 charset fallback in requests
	# https://github.com/kennethreitz/requests/issues/2086
	if "charset" in response["headers"]["Content-Type"]:
		possible_encoding = response["encoding"]
	else:
		possible_encoding = False

	markup = response["raw_content"]
	soup = BeautifulSoup(markup, "html5lib", from_encoding=possible_encoding)
	title_tag = soup.title

	if title_tag:
		title = title_tag.text.strip()

		if title.lower() in url:
			return None
		else:
			return title
	else:
		return None

	def posten_postnr_query(self, Address, Postort):
		url = 'http://www.posten.se/soktjanst/postnummersok/resultat.jspv?gatunamn=' + utility.escape(Address) + '&po=' + utility.escape(Postort)
	 
		response = utility.read_url(url)
		data = response["data"]
	 
		postnrs = {}
		for line in data.split("\n"):
			search = re.search('<TD class="firstcol">([^<]*)</TD><TD>([^<]*)</TD><TD>([^<]*)', line)
			if search:
				if postnrs.has_key(search.group(3)):
					postnrs[search.group(3)] += " & " + search.group(1) + " " + search.group(2)
				else:
					postnrs[search.group(3)] = search.group(1) + " " + search.group(2)
	 
		result = ""
		for postnr in postnrs.iterkeys():
			if len(result) != 0:
				result += ", "
			result += "%s: %s" % (postnr, postnrs[postnr])
			# print postnrs[postnr]
	 
		if len(result) == 0:
			return "no result :<"
		else:
			# print result
			return result

	def lookup_direct(self, reference):
		if reference.type == u"album":
			endpoint = u"v1/albums/{id}"
		elif reference.type == u"artist":
			endpoint = u"v1/artists/{id}"
		elif reference.type == u"playlist":
			endpoint = u"v1/users/{user_id}/playlists/{playlist_id}"
			return None  # Unsupported by this plugin
		elif reference.type == u"track":
			endpoint = u"v1/tracks/{id}"
		else:
			return None  # Unsupported by this plugin

		api_url = self.api_base_url + endpoint.format(id=reference.hash)
		response = utility.read_url(api_url)

		if not response:
			return None

		try:
			data = JSONDecoder().decode(response['data'])
		except ValueError:
			return None

		if data.get(u"status"):
			return None
		else:
			return self._format_result(reference.type, data)

def icq_lookup(icqid):
	url = 'http://www.icq.com/people/about_me.php?uin=' + utility.escape(icqid)
	response = utility.read_url(url)
	data = response["data"].replace("\n", "")

	m = re.search('<div class="uinf-2-2-2-1">(.*?)<\/div>.*?<div class="uinf-2-2-2-2">(.*?)<\/div>.*?<div class="uinf-2-2-2-4">(.*?)<\/div>.*?<div class="uinf-2-2-2-4">(.*?)<\/div>', data)

	if m:
		nick = m.group(1)
		info = m.group(2)
		if info:
			info = re.sub("\n|\r\n|\n\n",", ",info)
		city = m.group(3)
		country = m.group(4)
		
		if nick:
			result = nick
		if info or city or country:
			result = result + ": "
			
		if info:
			result = result + info
		if city:
			result = result + ", " + city
		if country:
			result = result + ", " + country
		return result
	else:
		return None

	def fetchFood(self, restaurant, day=None):
                if datetime.now() < datetime(2011, 8, 7):
                        return "Nien nien nien! http://www.youtube.com/watch?v=fkmv2eAESfM"
		if day == "today":
			return "Wiseass aren't you?"
		elif day != None:
			return "I'm sorry Dave Preston can only handle 'today'."
		

		response = utility.read_url(self.url)
		data = response["data"]

                lunches = re.findall("</b>\s*<BR>([^<]*)<", data)
                if lunches[0][0:6] == "Vi har":
                        lunches = lunches[1:]

                cnt = 1
		result = "Lunch Collegium "
			
		for lunch in lunches:
			result += str(cnt) + ": " + lunch + " "
			cnt += 1

		if result[-1:] == " ":
			result = result[:-1]

		if len(result) == 0 or len(lunches) == 0:
			return "No lunch available at %s ):" % restaurant
		else:
			return result

	def spot_lookup_direct(self, theSpot):
		url = "http://spotify.url.fi/%s/%s" % (theSpot.type, theSpot.hash)
		response = utility.read_url(url)
		data = response["data"]

		# Commence data mining

		artist = re.search(r"<span>Artist</span>\s*<a.*?>(?P<artist>.+?)</a>", data, re.DOTALL)
		if artist: artist = artist.group(1)

		album = re.search(r"<span>Album</span>\s*<a.+?>(?P<album>.+?)</a>", data, re.DOTALL)
		if album: album = album.group(1)

		year = re.search(r"<span>Year</span>\s*(?P<year>.+?)\s*</p>", data, re.DOTALL)
		if year: year = year.group(1)

		track = re.search(r"<span>Track</span>\s*<a.+?>(?P<track>.+?)</a>", data, re.DOTALL)
		if track: track = track.group(1)

		length = re.search("<span>Length</span>\s*(?P<length>.+?)\s*</p>", data, re.DOTALL)
		if length: length = length.group(1)

		output = "%s: %s | %s (%s)" % (artist, track, album, year)

		if not track:
			output = "%s: %s (%s)" % (artist, album, year)

		if not album:
			output = "%s" % artist

		if not artist:
			return "couldn't find shit, captain!"

		return output

	def trig_schema(self, bot, source, target, trigger, argument):
		if not argument:
			argument = self.id_presets.get(source.lower(), source.lower())
		else:
			argument = argument.strip().lower()
			self.id_presets[source.lower()] = argument
			self.save()

		if argument in self.id_directory:
			url = self.id_directory[argument]
			if isinstance(url, int):
				url = "http://timeedit.liu.se/4DACTION/iCal_downloadReservations/timeedit.ics?branch=5&id1=%d&lang=1" % url
			
			response = utility.read_url(url)

			parser = iCalParser()
			parser.process(response["data"])
			parser.events.sort()

			relevant_events = parser.events[0:7]
			event_outputs = []
			last_event = None
			for event in relevant_events:
				if last_event and last_event.start.day == event.start.day:
					event_outputs.append(event.short_description())
				else:
					event_outputs.append(event.long_description())
					
				last_event = event

			return "%s: %s" % (argument, " | ".join(event_outputs))
		else:
			return "I don't have the ID '" + argument + "' in my directory. Feel free to add it by typing .addschemaid <name> <url or timeedit id>."

	def query(self, argument):
		decoder = JSONDecoder()
		argument = utility.escape(argument)
		api_url = u"http://www.imdbapi.com/?t=%(search_term)s&r=json&plot=short" % \
				{"search_term": argument}
		site_search_url = u"http://akas.imdb.com/find?s=all&q=" + argument
		response = utility.read_url(api_url)

		if not response:
			return u"Couldn't connect to the API :( | Manual search: " + site_search_url

		try:
			data = decoder.decode(response['data'])
		except Exception:
			return u"Couldn't parse the API output :( | Manual search: " + site_search_url

		if data.get(u"Response") != u"True":
			return u"No results found! Maybe you should try searching manually: " + \
					site_search_url

		return \
				(u"%(title)s (%(year)s) - Rating: %(rating)s out of 10 - Genre: %(genre)s - " + \
				u"http://akas.imdb.com/title/%(id)s/ | More results: %(site_search_url)s") % \
				{u"title": data.get(u"Title", u"Missing title :S"),
					u"year": data.get(u"Year", u"Unknown year"),
					u"rating": data.get(u"Rating", u"N/A"),
					u"genre": data.get(u"Genre", u"Unknown"),
					u"id": data.get(u"ID", u"tt0107838"),
					u"site_search_url": site_search_url}

def posten_kolli_query(kolli_id):
    url = (
        "http://posten.se/tracktrace/TrackConsignments_do.jsp?trackntraceAction=saveSearch&consignmentId="
        + utility.escape(kolli_id)
    )
    response = utility.read_url(url)
    data = response["data"]

    search = re.search(
        "(?ims)<dt>Fr&aring;n:</dt><dd>(.*?)</dd>.*?rightcol.*h2>.*<h3>(.*?)</h3>\s*?(.*?)(<br/>|<div).*?<dt>Vikt:</dt><dd>(.*?)</dd>",
        data,
    )

    if search:
        sender = search.group(1)
        date = search.group(2)
        status = search.group(3)
        weight = search.group(5)

        if date and status:
            result = "%s fr\xe5n %s | %s: %s | %s" % (weight, sender, date, re.sub("<.+?>", "", status), url)
            return result
        else:
            return None
    else:
        return None

def imdb_info(url):
    response = utility.read_url(url)
    data = response["data"]

    m = re.search("<title>(.*?) \((\d+\/?I?)\)<\/title>", data)
    if m:
        title = m.group(1)
        year = m.group(2)
    else:
        title = ""
        year = ""

    m = re.search("<b>([0-9\.]+)\/10<\/b>", data)
    if m:
        rating = m.group(1)
    else:
        rating = 0

    m = re.search("<h5>Plot Summary:</h5> \n(.*?)\n<a", data)
    if m:
        tagline = '"%s" -' % m.group(1)
    else:
        m = re.search("<h5>Plot Outline:</h5> \n(.*?) <a", data)
        if m:
            tagline = '"%s" -' % m.group(1)
        else:
            tagline = ""

    m = re.findall('"\/Sections\/Genres\/(.*?)\/', data)
    if m:
        genres = ", ".join(set(m))
    else:
        genres = ""

    return "%s (%s) - Rating: %s/10 - Genre: %s - %s %s" % (title, year, rating, genres, tagline, url)

    def trig_temp(self, bot, source, target, trigger, argument):
        """ Usage: .temp [City] Uses data from temperature.nu, please direct all complaints to www.temperatur.nu """
        argument = argument.strip()
        if argument:
            argument = argument.strip()
            self.places[source] = argument
            self.save()
        else:
            if source in self.places:
                argument = self.places[source]
            else:
                argument = "ryd"

        argument_text = argument
        argument = utility.asciilize(argument)
        argument = utility.escape(argument)

        # awesome hack to include avesta!
        if argument.lower() == "avesta":
            actual_argument = "fors"
        else:
            actual_argument = argument

        url = "http://www.temperatur.nu/termo/%s/temp.txt" % actual_argument.lower()
        response = utility.read_url(url)
        m = None

        if response:
            data = response["data"]
            m = _get_temp_re.match(data)

        if m and m.group(1) != "not found":
            return "Temperature in %s: %s." % (argument_text, m.group(1))
        else:
            return "Temperature in %s: invalid place, try using .yr instead." % (argument_text)