Python Util.urlopen方法代码示例

# 需要导入模块: from util import Util [as 别名]
# 或者: from util.Util import urlopen [as 别名]
    def login(self, username, password):
        """Logs the user in.

        Downloads the relevant cookies to keep the user logged in."""

        logging.info("Logging in...")

        rawPage = Util.urlopen(self._getURL("loginPage"))
        if not rawPage:
            logging.error("Cannot load login page.")
            return False
        soup = BeautifulSoup(rawPage.read())

        logging.debug("Checking for previous login.")
        logged = self.getLoggedUser(soup)
        if logged:
            if logged == username:
                logging.info("Already logged as %s.", logged)
                self.loggedIn = True
                return True
            else:
                logging.info("Already logged as %s, but want to log in as %s.", logged, username)
                self.logout()

        # continue logging in
        postValues = dict()
        logging.debug("Assembling POST data.")

        # login fields
        loginElements = soup("input", type=["text", "password", "checkbox"])
        for field, val in zip(loginElements, [username, password, 1]):
            postValues[field["name"]] = val

        # other nescessary fields
        for field in soup("input", type=["hidden", "submit"]):
            postValues[field["name"]] = field["value"]

        url = self._getURL("loginPage")

        # login to the site
        logging.debug("Submiting login form.")
        rawPage = Util.urlopen(url, postValues)
        if not rawPage:
            logging.error("Cannot load response after login.")
            return False
        soup = BeautifulSoup(rawPage.read())

        logging.debug("Checking the result.")
        if self.getLoggedUser(soup):
            logging.info("Logged in successfully as %s.", username)
            self.loggedIn = True
            return True
        else:
            logging.error("Cannot logon to the site (probably wrong username or password).")
            return False

# 需要导入模块: from util import Util [as 别名]
# 或者: from util.Util import urlopen [as 别名]
    def loadCacheQuick(self, wp):
        """Loads details from map server.

        Loads just basic cache details, but very quickly"""

        if not self.loggedIn or not (isinstance(wp, StringTypes) and wp.startswith("GC")):
            return

        logging.info("Loading quick details about %s...", wp)

        # assemble request
        params = urlencode({"i": wp})
        url = self.mapurl + "?" + params

        # make request
        rawPage = Util.urlopen(url)
        if not rawPage:
            logging.error("Cannot load search page.")
            return
        res = json.loads(rawPage.read())

        # check for success
        if res["status"] != "success":
            return None
        data = res["data"][0]

        # prettify some data
        size = data["container"]["text"].lower()
        hidden = datetime.strptime(data["hidden"], '%m/%d/%Y').date()

        # assemble cache object
        c = Cache(wp, data["name"], data["type"]["text"], None, data["available"], None,
            size, data["difficulty"]["text"], data["terrain"]["text"],
            data["owner"]["text"], hidden, None)
        return c

# 需要导入模块: from util import Util [as 别名]
# 或者: from util.Util import urlopen [as 别名]
    def getLoggedUser(self, soup=None):
        """Returns the name of curently logged user.

        Or None if no user is logged in."""

        if not isinstance(soup, BeautifulSoup):
            logging.debug("No 'soup' passed, loading login page on my own.")
            page = Util.urlopen(self._getURL("loginPage"))
            soup = BeautifulSoup(page.read())

        logging.debug("Checking for already logged user.")
        try:
            return soup.find("div", "LoggedIn").find("strong").text
        except AttributeError:
            return None

# 需要导入模块: from util import Util [as 别名]
# 或者: from util.Util import urlopen [as 别名]
    def loadCache(self, wp):
        """Loads details from cache page.

        Loads all cache details and return fully populated cache object."""

        if not self.loggedIn or not (isinstance(wp, StringTypes) and wp.startswith("GC")):
            return

        logging.info("Loading details about %s...", wp)

        # assemble request
        params = urlencode({"wp": wp})
        url = self._getURL("cacheDetails") + "?" + params

        # make request
        rawPage = Util.urlopen(url)
        if not rawPage:
            logging.error("Cannot load search page.")
            return
        soup = BeautifulSoup(rawPage)

        # parse raw data
        cacheDetails = soup.find(id="cacheDetails")
        name = cacheDetails.find("h2")
        cacheType = cacheDetails.find("img").get("alt")
        author = cacheDetails("a")[1]
        hidden = cacheDetails.find("div", "minorCacheDetails").findAll("div")[1]
        location = soup.find(id="uxLatLon")
        state = soup.find("ul", "OldWarning")
        found = soup.find("div", "FoundStatus")
        DandT = soup.find("div", "CacheStarLabels").findAll("img")
        size = soup.find("div", "CacheSize").find("img")
        attributesRaw = soup("div", "CacheDetailNavigationWidget")[0].findAll("img")
        userContent = soup("div", "UserSuppliedContent")
        hint = soup.find(id="div_hint")
        favorites = soup.find("span", "favorite-value")

        # prettify data
        name = name.text.encode("utf-8", "xmlcharrefreplace")
        author = author.text.encode("utf-8", "xmlcharrefreplace")
        hidden = datetime.strptime(hidden.text.split()[2], '%m/%d/%Y').date()
        try:
            lat, lon = Util.parseRaw(location.text)
            location = geo.Point(Util.toDecimal(*lat), Util.toDecimal(*lon))
        except ValueError:
            loggin.debug("Could not parse coordinates")
        state = state is None
        found = found and "Found It!" in found.text or False
        dif, ter = map(lambda e: float(e.get("alt").split()[0]), DandT)
        size = " ".join(size.get("alt").split()[1:]).lower()
        attributesRaw = map(lambda e: e.get("src").split('/')[-1].rsplit("-", 1), attributesRaw)
        attributes = {} # parse attributes by src to know yes/no
        for attribute_name, appendix in attributesRaw:
            if appendix.startswith("blank"):
                continue
            attributes[attribute_name] = appendix.startswith("yes")
        summary = userContent[0].text.encode("utf-8", "xmlcharrefreplace")
        description = userContent[1]
        hint = Util.rot13(hint.text.strip().encode('utf-8'))
        favorites = int(favorites.text)

        # assemble cache object
        c = Cache(wp, name, cacheType, location, state, found,
            size, dif, ter, author, hidden, attributes,
            summary, description, hint, favorites)
        return c

# 需要导入模块: from util import Util [as 别名]
# 或者: from util.Util import urlopen [as 别名]
    def _searchGetPage(self, point, page=1):
        """Returns one page of caches as a list.

        Searches for a caches around a point and returns N-th page (specifiend by page argument)."""

        if not self.loggedIn or not isinstance(point, geo.Point) or type(page) is not IntType:
            return

        logging.info("Fetching page %d.", page)

        # assemble request
        params = urlencode({"lat": point.latitude, "lng": point.longitude})
        url = self._getURL("cachesNearest") + "?" + params

        # we have to add POST for other pages than 1st
        if page == 1:
            data = None
        else:
            # TODO handle searching on second page without first
            data = self.paggingHelpers
            data["__EVENTTARGET"] = self.paggingPostbacks[page]
            data["__EVENTARGUMENT"] = ""

        # make request
        rawPage = Util.urlopen(url, data)
        if not rawPage:
            logging.error("Cannot load search page.")
            return list()
        soup = BeautifulSoup(rawPage)

        # root of a few following elements
        pageBuilders = soup("td", "PageBuilderWidget")

        # parse pagging widget
        total, page, pageCount = map(lambda elm: int(elm.text), pageBuilders[0].findAll("b"))
        logging.debug("Found %d results. Showing page %d of %d.", total, page, pageCount)

        # save search postbacks for future usage
        if page == 1:
            paggingLinks = filter(lambda e: e.get("id"), pageBuilders[1].findAll("a"))
            paggingPostbacks = {int(link.text): link.get("href").split("'")[1] for link in paggingLinks}
            self.paggingPostbacks = paggingPostbacks

            # other nescessary fields
            self.paggingHelpers = dict()
            for field in soup("input", type="hidden"):
                self.paggingHelpers[field["name"]] = field["value"]

        # parse results table
        data = soup.find("table", "SearchResultsTable").findAll("tr", "Data")
        result = []
        for cache in data:

            # parse raw data
            typeLink, nameLink = cache("a", "lnk")
            direction, info, DandT, placed, lastFound = cache("span", "small")
            found = cache.find("img", title="Found It!") is not None
            size = cache.find("td", "AlignCenter").find("img")
            author, wp, area = map(lambda t: t.strip(), info.text.split("|"))

            # prettify data
            cacheType = typeLink.find("img").get("alt")
            name = nameLink.span.text.strip().encode("ascii", "xmlcharrefreplace")
            state = not "Strike" in nameLink.get("class")
            size = " ".join(size.get("alt").split()[1:]).lower()
            dif, ter = map(float, DandT.text.split("/"))
            hidden = datetime.strptime(placed.text, '%m/%d/%Y').date()
            author = author[3:].encode("ascii", "xmlcharrefreplace") # delete "by "

            # assemble cache object
            c = Cache(wp, name, cacheType, None, state, found, size, dif, ter, author, hidden)
            logging.debug("Parsing cache: %s", c)
            result.append(c)

        return result