Python client.getPage函数代码示例

def job():
    logger.info("Getting page")
    txt = yield client.getPage(settings.url)
    parsed_txt = html.fromstring(txt)
    # pylint:disable=E1102
    el = sel_base(parsed_txt)
    # if element does not exist
    if not len(el):
        return
    chapter_name = sel_chapter_name(el[0])[0].text.strip()
    update_date = sel_update_date(el[0])[0].text.strip()

    # do nothing if no new chapter avaliable
    if settings.current_chapter == chapter_name:
        logger.info("Current chapter is %s, not a new chapter, do nothing.", chapter_name)
        return
    else:
        # saves the new chapter name to settings
        settings.current_chapter = chapter_name

    data = {
        "chapter_name": chapter_name,
        "update_date": update_date
    }
    logger.info("{\n\tchapter_name: %s,\n\tupdate_date: %s\n}", chapter_name, update_date)

    logger.info("Sending data to webhook: %s", settings.webhook_url)
    try:
        yield client.getPage(settings.webhook_url,
                            method="POST", postdata=json.dumps(data),
                            headers={"Content-Type": "application/json; chatset=utf-8"})
    except Exception as exc:
        logger.error("Error occured while sending data.\n%s", str(exc))
    else:
        logger.info("Data sent")

		def getFromXunlei(result,parameter):
			header,tailer,filename = parameter
			url = 'http://bt.box.n0808.com/%s/%s/%s' % (header,tailer,filename)
			self.transport.write('start downloading from %s\n' % url)
			getPage(url,filename).addCallbacks(
				ParseTorrent,
				lambda result:self.transport.write('download torrent failed\n'))

	def startRun(self):
		if self.weatherPluginEntry is not None:
			self["statustext"].text = _("Getting weather information...")
			url = "http://weather.service.msn.com/data.aspx?weadegreetype=%s&culture=%s&wealocations=%s" % (self.weatherPluginEntry.degreetype.value, self.language, self.weatherPluginEntry.weatherlocationcode.value)
			getPage(url).addCallback(self.xmlCallback).addErrback(self.error)
		else:
			self["statustext"].text = _("No locations defined...\nPress 'Menu' to do that.")

    def openTest(self):
                self["info"].setText("Downloading list...")
                testno = 1

                xurl = "http://www.et-view-support.com/addons/XTA-team/" + self.addon + "/list.txt"
                print "xurl =", xurl
                getPage(xurl).addCallback(self.gotPage).addErrback(self.getfeedError)

	def monsteruploads(self, data, url):
		if not re.match('.*?eval\(function\(p\,a\,c\,k\,e\,d', data, re.S):
			id = re.findall('type="hidden" name="id".*?value="(.*?)"', data, re.S)
			fname = re.findall('type="hidden" name="fname".*?value="(.*?)"', data, re.S)
			referer = re.findall('type="hidden" name="referer".*?value="(.*?)"', data, re.S)
			info = urlencode({
				'op':           "download2",
				'usr_login':    "",
				'id':           id[0],
				'fname':        fname[0],
				'referer':      "",
				'method_free':  "Kostenloser Download"
				})
			getPage(url, method='POST', postdata = info, headers={'Content-Type':'application/x-www-form-urlencoded'}).addCallback(self.monsteruploads_post).addErrback(self.errorload)
			
		else:
		
			get_packedjava = re.findall("<script type=.text.javascript.>eval.function(.*?)</script>", data, re.S|re.DOTALL)
			if get_packedjava:
				sJavascript = get_packedjava[1]
				sUnpacked = cJsUnpacker().unpackByString(sJavascript)
				if sUnpacked:
					stream_url = re.findall('type="video/divx"src="(.*?)"', sUnpacked)
					if stream_url:
						print stream_url[0]
						self._callback(stream_url[0])		
					else:
						self.stream_not_found()
				else:
					self.stream_not_found()
			else:
				self.stream_not_found()

	def getCookie(self, data=None):
		#url = "http://www.skygo.sky.de/index.html"
		url = "http://www.skygo.sky.de/skyguide"
		msg_log = "get page: '{0}'".format(url)
		print msg_log
		self.addLog(msg_log)
		getPage(url, headers=self.headers1, agent=self.agent, timeout=30, cookies=self.ck).addCallback(self.getSkyGuidePages).addErrback(self.dataError)

	def	request(self):
		# if self.serien_name in self.termineCache:
		# 	self.__callback(self.termineCache[self.serien_name], False)
		# else:
		print "[SerienRecorder] suche ' %s '" % self.serien_name
		print self.serie_url
		getPage(getURLWithProxy(self.serie_url), agent=getUserAgent(), headers=getHeaders()).addCallback(self.__callback).addErrback(self.__errback)

 def recaptcha(self, data):
     if data.find('http://goo.gl') != -1:
         self.mod.on_recaptcha(self.index, data[data.index('"id"'):].split('"')[3])
     else:
         self.challenge = data[data.index('challenge'):].split('\'')[1]
         url = '{"longUrl": "http://www.google.com/recaptcha/api/image?c=%s"}' % self.challenge
         getPage("https://www.googleapis.com/urlshortener/v1/url", method="POST", postdata=url, headers={'Content-Type': 'application/json'}).addCallback(self.recaptcha)

    def start(self):
        try:
            data = yield getPage("%slogin/" % self.config['url'], cookies=self.cookies)
        except Exception:
            raise

        result = CSRF_RE.search(data)
        if not result:
            print "error"
            return
        token = result.group(1)

        # login
        try:
            data = yield getPage("%slogin/" % self.config['url'],
                                 method='POST',
                                 postdata=urllib.urlencode({
                                     'csrfmiddlewaretoken': token,
                                     'username': self.config['username'],
                                     'password': self.config['password'],
                                 }),
                                 cookies=self.cookies,
                                 headers={'Content-Type': 'application/x-www-form-urlencoded'})
        except Exception:
            raise

        self.task = task.LoopingCall(self.update)
        self.task.start(self.config["update_rate"])

    def updateTimer(self):
        reactor.callLater(UPDATETIMER, self.updateTimer)

        def setTreeStatus(result):
            r = json.loads(result)
            status = r[u"status"]
            treename = r[u"tree"]
            changed = False
            logmsg = "Regular Tree check says: {} is {}".format(treename, status)
            if not treename in self.statusCache:
                changed = True
            if treename in self.statusCache and status != self.statusCache[treename]:
                changed = True
            if changed:
                # if status previously unknown (= bot has just started)
                # or a changed status then set the topic
                channel = tree2channel[tree]
                topic = "{} is {}!".format(treename, status)
                self.topic(channel, topic)
                logmsg += ", which is new. changing topic."
                print logmsg
            else:
                logmsg += ", just like before"
                print logmsg
            self.statusCache[treename] = status

        for tree in tree2channel:
            url = URL.format(tree)
            getPage(url).addCallbacks(callback=setTreeStatus)

    def test_start_stop(self):
        yield self._storage.put("abc", StringIO("hello world"))
        yield self._server.start()
        # Starting multiple times is fine.
        yield self._server.start()
        storage_url = yield self._storage.get_url("abc")

        # It might not have started actually accepting connections yet...
        yield self.wait_for_server(self._server)
        self.assertEqual((yield getPage(storage_url)), "hello world")

        # Check that it can be killed by the current user (ie, is not running
        # as root) and still comes back up
        old_pid = yield self._server.get_pid()
        os.kill(old_pid, signal.SIGKILL)
        new_pid = yield self._server.get_pid()
        self.assertNotEquals(old_pid, new_pid)

        # Give it a moment to actually start serving again
        yield self.wait_for_server(self._server)
        self.assertEqual((yield getPage(storage_url)), "hello world")

        yield self._server.stop()
        # Stopping multiple times is fine too.
        yield self._server.stop()

    def delete(self):
        """
        Delete the specific workflow from VelesForge. Requires write access.
        """
        if not self._confirm():
            return
        print("Deleting %s..." % self.name)
        sys.stdout.flush()

        def finished(response):
            print("Successfully deleted %s" % self.name)
            sys.stdout.flush()
            self.stop()

        def failed(failure):
            try:
                failure.raiseException()
            except:
                self.exception("Failed to delete %s:", self.name)
            self.stop(failure, False)

        url = "%s%s?query=delete&name=%s&token=%s" % (
            self.base, root.common.forge.service_name, self.name, self.id)
        self.debug("Requesting %s", url)
        getPage(url.encode('charmap')).addCallbacks(callback=finished,
                                                    errback=failed)

    def list(self):
        """
        Print the list of available workflows on VelesForge.
        """
        def finished(response):
            self.debug("Received %s", response)
            try:
                response = json.loads(response.decode('UTF-8'))
            except ValueError as e:
                self.exception("Failed to parse the response from server: %s",
                               response)
                self.stop(Failure(e), False)
                return
            table = PrettyTable("Name", "Description", "Author", "Version",
                                "Date")
            table.align["Name"] = table.align["Description"] = 'l'
            for item in response:
                table.add_row(*item)
            print(table)
            sys.stdout.flush()
            self.stop()

        def failed(failure):
            self.return_code = 1
            try:
                failure.raiseException()
            except:
                self.exception("Failed to list the available models:")
            self.stop(failure, False)

        url = self.base + root.common.forge.service_name + "?query=list"
        self.debug("Requesting %s", url)
        getPage(url.encode('charmap')).addCallbacks(callback=finished,
                                                    errback=failed)

    def __init__(self, session):
        Screen.__init__(self, session)
        self["info"] = Label('Ładuję...')
        self["myPic"] = Pixmap()
        try:
            os.unlink('/tmp/meteo.png')
        except:
            pass
        self["myActionMap"] = ActionMap(["MeteoActions"],
        {
            "ok": self.cancel,
            "cancel": self.cancel,
            "menu": self.menu,
            "blue": self.menu,
            "green": self.next,
            "left": self.prev,
            "right": self.next,
            "start": self.start,
            "info": self.ShowOpis,
            "yellow": self.ShowOpis,
            "list": self.choose,
        }, -1)

        getPage('http://e2.areq.eu.org/ummeteo/version').addCallback(self.updateCB).addErrback(self.errorUpdate)

        self.onLayoutFinish.append(self.layoutFinished)

	def handleWebsite(self, website):
		info(website.getAttribute("name"))
		if self.number[:2] == "00":
			number = website.getAttribute("prefix") + self.number.replace(self.countrycode, "")
		else:
			number = self.number

		url = website.getAttribute("url")
		if re.search('$AREACODE', url) or re.search('$PFXAREACODE', url):
			error("(PFX)ARECODE cannot be handled")
			# self.caller = _("UNKNOWN")
			self.notifyAndReset()
			return
		#
		# Apparently, there is no attribute called (pfx)areacode anymore
		# So, this below will not work.
		#
		if re.search('\\$AREACODE', url) and website.hasAttribute("areacode"):
			areaCodeLen = int(website.getAttribute("areacode"))
			url = url.replace("$AREACODE", number[:areaCodeLen]).replace("$NUMBER", number[areaCodeLen:])
		elif re.search('\\$PFXAREACODE', url) and website.hasAttribute("pfxareacode"):
			areaCodeLen = int(website.getAttribute("pfxareacode"))
			url = url.replace("$PFXAREACODE", "%(pfxareacode)s").replace("$NUMBER", "%(number)s")
			url = url % {'pfxareacode': number[:areaCodeLen], 'number': number[areaCodeLen:]}
		elif re.search('\\$NUMBER', url):
			url = url.replace("$NUMBER", "%s") % number
		else:
			error("cannot handle websites with no $NUMBER in url")
			# self.caller = _("UNKNOWN")
			self.notifyAndReset()
			return
		info("Url to query: " + url)
		url = url.encode("UTF-8", "replace")
		self.currentWebsite = website
		getPage(url, agent="Mozilla/5.0 (Windows; U; Windows NT 6.0; de; rv:1.9.0.5) Gecko/2008120122 Firefox/3.0.5").addCallback(self._gotPage).addErrback(self._gotError)