Python Download.getSOURCE方法代码示例

# 需要导入模块: from Download import Download [as 别名]
# 或者: from Download.Download import getSOURCE [as 别名]
 def doRequest(self):
   d = Download(self.Url)
   if d.doRequest():
     return 1
   
   self.recs = d.getSOURCE()
   return 0

# 需要导入模块: from Download import Download [as 别名]
# 或者: from Download.Download import getSOURCE [as 别名]
def isGoogleSearch(schema, ip):
  d = Download(schema + '://' + ip)
  if d.doRequest():
    return False

  if Utility.containsGoogle(d.getSOURCE()):
    return True
  
  return False

# 需要导入模块: from Download import Download [as 别名]
# 或者: from Download.Download import getSOURCE [as 别名]
    def run(self):
        url = self.BASE_URL + self.SeasonId + self.BASE_URL_PART_3 + str(self.PageNumber) + self.BASE_URL_PART_5
        d = Download(url)
        if d.doRequest():
            # fail
            print 'ERROR: ' + self.SeasonId + '-' + str(self.PageNumber)
        else:
            utfstr2file(d.getSOURCE(), './data/' + self.SeasonId + '-' + str(self.PageNumber) + '.raw')

        return url

# 需要导入模块: from Download import Download [as 别名]
# 或者: from Download.Download import getSOURCE [as 别名]
    def requestHtml(self):
        url = self.BaseUrl + self.ISBN
        # print url, self.User_Agent
        d = Download(url, self.User_Agent)
        if d.doRequest():
            return 1

        self.HTML = d.getSOURCE()

        return 0

# 需要导入模块: from Download import Download [as 别名]
# 或者: from Download.Download import getSOURCE [as 别名]
    def request(self):
        baseUrl = "http://shaishufang.com/index.php/site/main/uid/"
        postFix = "/friend/false/category//status//type//page/"
        url = baseUrl + self.UID + postFix + str(self.Page)

        d = Download(url, self.Cookie, self.Proxy)
        if d.doRequest():
            return False

        self.HTML = d.getSOURCE()
        return True

# 需要导入模块: from Download import Download [as 别名]
# 或者: from Download.Download import getSOURCE [as 别名]
    def request(self):
        baseUrl = 'http://shaishufang.com/index.php/site/detail/uid/'
        postFix = '/status//category/none/friend/false'
        url = baseUrl + self.UID + '/ubid/' + self.BID + postFix

        d = Download(url, self.Cookie, self.Proxy)
        if d.doRequest():
            return False

        self.HTML = d.getSOURCE()
        return True

# 需要导入模块: from Download import Download [as 别名]
# 或者: from Download.Download import getSOURCE [as 别名]
  def run(self):
    while True:
      print 'INFO: ........................................ START'
      stats = self.dbm.getStats()
      print 'INFO: deadLinks-', stats[0], ' unvisitedLinks-', stats[1], ' visitedLinks-', stats[2]
      # get an url from unvisitedLinks
      url = self.dbm.retrieveUnvisitedLink()
      if url == False:
        print 'DEBUG: DONE -- retrieveUnvisitedLink return False'
        break

      print 'DEBUG: Processing ', url

      if not self.urlFilter.isPlainText(url):
        print 'DEBUG: NotPlainTextURL ', url
        continue
      
      if not self.domainFilter.isInDomain(url):
        print 'DEBUG: NOT IN DOMAIN ', url
        continue

      # requet the url
      d = Download(url)
      if d.doRequest() == 1:
        if not self.dbm.createDeadLink(url):
          print 'DEBUG: deadLinks already contain ', url
        else:
          print 'DEBUG: Add To deadLinks ', url
      else:
        if self.dbm.createVisitedLink(url):
          print 'DEBUG: Add To visitedLinks ', url
        else:
          print 'DEBUG: Failed Add To visitedLinks ', url

        # extract urls from the sourc2
        u = URLExtractor(d.getSOURCE(), url)
        tmpUrls = u.getUrls()
        if tmpUrls:
          for url in tmpUrls:
            if self.dbm.isInDeadLink(url):
              continue
            elif self.dbm.isInVisitedLink(url):
              continue
            elif self.dbm.isInUnvisitedLink(url):
              continue
            else:
              print 'DEBUG: Add To unvisitedLink ', url
              self.dbm.createUnvisitedLink(url)
    
      print 'INFO: ........................................ END'

# 需要导入模块: from Download import Download [as 别名]
# 或者: from Download.Download import getSOURCE [as 别名]
    def run(self, processName='MainProcess'):
        for isbn in self.ISBNS:
            url = 'http://www.amazon.cn/s/ref=nb_sb_noss?field-keywords=' + isbn
            d = Download(url)
            if d.doRequest():
                print 'ERROR[' + processName + ']: ', isbn, 'NERR'
                appendstr2file(isbn, './NERR.txt')
                continue

            asin = ASINParser(d.getSOURCE())
            if asin.getAsin():
                print 'INFO[' + processName + ']: ', isbn, asin.getAsin()
                appendstr2file(isbn + ',' + asin.getAsin(), './OK.txt')
            else:
                print 'WARN[' + processName + ']: ', isbn, 'NOER'
                appendstr2file(isbn, './NOER.txt')

# 需要导入模块: from Download import Download [as 别名]
# 或者: from Download.Download import getSOURCE [as 别名]
def worker(appids, isbns, appidsCycle):
    # appidsCycle = cycle(appids)

    for isbn in isbns:
        url = 'http://' + appidsCycle.next() + '.appspot.com/url?url=' + 'http://book.douban.com/isbn/' + str(isbn)
        # print 'DEBUG: ', url

        d = Download(url)
        if d.doRequest():
            print isbn, 'network error'
            continue

        j = json.loads(d.getSOURCE())
        print isbn, j['status_code']

    return

# 需要导入模块: from Download import Download [as 别名]
# 或者: from Download.Download import getSOURCE [as 别名]
def Google_Web_Search_Helper(q, hl='en', start=0):
  Google_Web_Search_URL = 'https://www.google.com/search?'

  if not q:
    return {}
  else:
    Google_Web_Search_URL = Google_Web_Search_URL + 'q=' + q

  Google_Web_Search_URL = Google_Web_Search_URL + '&hl=' + hl
  Google_Web_Search_URL = Google_Web_Search_URL + '&start=' + start

  d = Download(Google_Web_Search_URL)
  if d.doRequest():
    return {}
  else:
    g = GoogleSearchResultParser(d.getSOURCE())
    return g.getJson()
  """

# 需要导入模块: from Download import Download [as 别名]
# 或者: from Download.Download import getSOURCE [as 别名]
    def run(self, processName='MainProcess'):
        for asin in self.ASINS:
            url = 'http://www.amazon.cn/dp/' + asin
            d = Download(url)
            if d.doRequest():
                print 'ERROR[' + processName + ']: ', asin, 'NERR'
                appendstr2file(asin, './NERRBasicInfo.txt')
                continue

            b = BasicInfoParser(d.getSOURCE())
            jsonRes = b.basicInfo()

            if json.loads(jsonRes):
                print 'info[' + processName + ']: ', asin
                appendstr2file(jsonRes, './OKBasicInfo.txt')
            else:
                print 'WARN[' + processName + ']: ', asin, 'NOER'
                appendstr2file(asin, './NOERBasicInfo.txt')

# 需要导入模块: from Download import Download [as 别名]
# 或者: from Download.Download import getSOURCE [as 别名]
  def walker(self):
    while True:
      urls = self.dbm.retrieveUnvisitedLinks(0, 100)
      urls = self.urlFilter.getFilteredUrls(urls)
      if len(urls) == 0:
        break

      for url in urls:
        print 'INFO: Processing ', url
        d = Download(url)
        if d.doRequest() == 1:
          self.dbm.createDeadLink(url)
        else:
          self.dbm.createVisitedLink(url)
          u = URLExtractor(d.getSOURCE(), url)
          tmpUrls = u.getUrls()
          if tmpUrls:
            self.dbm.createUnvisitedLinks(list(set(tmpUrls)))

    return True

# 需要导入模块: from Download import Download [as 别名]
# 或者: from Download.Download import getSOURCE [as 别名]
  def getStats(self):
    d = Download(self.API)
    if d.doRequest():
      return False

    res = []
    j = loads(d.getSOURCE())
    for item in j['resultSets'][0]['rowSet']:
      tmp = []
      name = item[3]
      pos = item[5]
      if item[6] == 'null':
        height = 'None'
      else:
        height = item[6]
      if item[7] == " ":
        weight = 'None'
      else:
        weight = item[7]
      age = item[9]
      if item[10] == 'R' or item[10] == 'None' or item[10] == None:
        exp = 0
      else:
        exp = item[10]

      tmp.append(name)
      tmp.append(pos)
      tmp.append(height)
      tmp.append(weight)
      tmp.append(age)
      tmp.append(exp)
      res.append(tmp)

    if len(res) == 0:
      return False
    else:
      return res

# 需要导入模块: from Download import Download [as 别名]
# 或者: from Download.Download import getSOURCE [as 别名]
#!/usr/bin/env python
#coding=utf-8
#
# Author: Archer Reilly
# Date: 11/Aug/2014
# File: PlayerInfoParserTest.py
# Description: test the PlayerInfoParser class
# Website: http://csrgxtu.blog.com/
#
# Produced By CSRGXTU
from PlayerInfoParser import PlayerInfoParser
from Download import Download

URL = "http://sports.qq.com/d/f_players/3/2890/"
player = Download(URL)
if player.doRequest() != 0:
  print "Download Cant Do Requst"
else:
  print "Successfully Do Request"

playerParser = PlayerInfoParser(player.getSOURCE())

# 需要导入模块: from Download import Download [as 别名]
# 或者: from Download.Download import getSOURCE [as 别名]
 def doRequest(self, url):
   d = Download(url)
   if d.doRequest() == None:
     return None
   else:
     return d.getSOURCE()