Python urlfetch.fetch函数代码示例

 def get(self, platform, soldier):
     ua = 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.6; en-US; rv:1.9.2.13) Gecko/20101203 Firefox/3.6.13'
     referer = 'http://bfbcs.com/' + platform
     cache_tag = 'bfbcs::' + platform + '/' + soldier
     raw = memcache.get(cache_tag)
     url = 'http://bfbcs.com/stats_' + platform + '/' + soldier
     if raw is None:
         response = urlfetch.fetch(url, headers={'User-Agent' : ua, 'Referer' : referer })
         raw = response.content
         memcache.set(cache_tag, raw, 600)
     pcode = re.findall('([a-z0-9]{32})', raw)
     self.response.out.write('<strong>PCODE</strong> ' + str(pcode[0]) + '<br />')
     if len(pcode) == 1:
         pcode = pcode[0]
         payload = 'request=addplayerqueue&pcode=' + pcode
         self.response.out.write('<strong>PAYLOAD</strong> ' + payload + ' (' + str(len(payload))+ ' bytes)<br />')
         headers = {'User-Agent' : ua, 'Referer' : url, 'X-Requested-With' : 'XMLHttpRequest', 'Content-Type' : 'application/x-www-form-urlencoded; charset=UTF-8', 'Content-Length' : '61', 'Accept' : 'application/json, text/javascript, */*', 'Accept-Language' : 'en-us,en;q=0.5', 'Accept-Encoding' : 'gzip,deflate', 'Accept-Charset' : 'ISO-8859-1,utf-8;q=0.7,*;q=0.7', 'Keep-Alive' : 115, 'Host' : 'bfbcs.com', 'Pragma' : 'no-cache', 'Cache-Control' : 'no-cache', 'Cookie' : '__utma=7878317.1843709575.1297205447.1298572822.1298577848.12; __utmz=7878317.1297205447.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none); sessid=enqd028n30d2tr4lv4ned04qi0; __utmb=7878317.21.10.1298577848; __utmc=7878317' }
         response = urlfetch.fetch(url, payload=payload, headers=headers, method='POST')
         if response.status_code == 500:
             response = urlfetch.fetch(url, payload=payload, headers=headers, method='POST')
             if response.status_code == 500:
                 self.write('<strong>FAILED</strong>')
             else:
                 self.write('<strong>RESULT</strong> OK ' + response.content)
         else:
             self.write('<strong>RESULT</strong> OK ' + response.content)

    def login(self):
        response = fetch("http://m.facebook.com/")

        m = re.search('''name="post_form_id" value="([^"]+?)"''', response.body)
        self.post_form_id = m.group(1)

        response = fetch(
            "https://www.facebook.com/login.php?m=m&refsrc=http%3A%2F%2Fm.facebook.com%2F&refid=0",
            data={
                "lsd": "off",
                "charset_test": "€,´,€,´,水,Д,Є",
                "version": "1",
                "ajax": "1",
                "width": "1280",
                "pxr": "1",
                "email": self.username,
                "pass": self.password,
                "submit": "Log In",
                "post_form_id": self.post_form_id,
            },
            headers={"Referer": "http://m.facebook.com/"},
        )

        set_cookie = response.getheader("Set-Cookie")
        self.cookies = sc2cs(set_cookie)

        url = response.getheader("location")
        response = fetch(url, headers={"Referer": "http://m.facebook.com/", "Cookie": self.cookies})

        self.post_form_id = re.search('''name="post_form_id" value="([^"]+?)"''', response.body).group(1)
        self.fb_dtsg = re.search('''name="fb_dtsg" value="([^"]+?)"''', response.body).group(1)

        return response

    def login(self):
        response = fetch(
            "http://m.douban.com/"
        )

        set_cookie = response.getheader('Set-Cookie')
        self.cookies = sc2cs(set_cookie)

        self.session = response.getheader('location').split('=', 1)[-1]

        response = fetch(
            "http://m.douban.com/",
            data = {
                'form_email': self.username,
                'form_password': self.password,
                'redir': '',
                'user_login': '登录',
                'session': self.session
            },
            headers = {
                'Referer': 'http://m.douban.com/',
                'Cookie': self.cookies,
            }
        )
        set_cookie = response.getheader('Set-Cookie')
        self.cookies += "; " + sc2cs(set_cookie)

        return response

def resolve_url(url):
  headers=HTTP_DESKTOP_UA
  cookie = Cookie.SimpleCookie()
  
  form_fields = {
   "inputUserName": __settings__.getSetting('username'),
   "inputPassword": __settings__.getSetting('password')
  }

  form_data = urllib.urlencode(form_fields)

  response = urlfetch.fetch(
    url = 'http://4share.vn/?control=login',
	method='POST',
    headers = headers,
	data=form_data,
    follow_redirects = False)

  cookie.load(response.headers.get('set-cookie', ''))
  headers['Cookie'] = _makeCookieHeader(cookie)
  
  response = urlfetch.fetch(url,headers=headers, follow_redirects=True)

  soup = BeautifulSoup(response.content, convertEntities=BeautifulSoup.HTML_ENTITIES)
  for item in soup.findAll('a'):
    if item['href'].find('uf=')>0:
      url=item['href']

  if url.find('uf=')<0:
    xbmc.executebuiltin((u'XBMC.Notification("%s", "%s", %s)' % ('Authentication', 'Please check your 4share username/password', '15')).encode("utf-8"))   
    return
	  
  item = xbmcgui.ListItem(path=url)
  xbmcplugin.setResolvedUrl(int(sys.argv[1]), True, item)

def pub2fanfou(username, password, status):
    # 获取表单token
    response = urlfetch.fetch("http://m.fanfou.com/")
    token = re.search('''name="token".*?value="(.*?)"''', response.body).group(1)

    # 登录
    response = urlfetch.fetch(
        "http://m.fanfou.com/",
        data={"loginname": username, "loginpass": password, "action": "login", "token": token, "auto_login": "on"},
        headers={"Referer": "http://m.fanfou.com/"},
    )

    # cookies
    cookiestring = response.cookiestring
    print cookiestring

    # 获取表单token
    response = urlfetch.fetch(
        "http://m.fanfou.com/home", headers={"Cookie": cookiestring, "Referer": "http://m.fanfou.com/home"}
    )
    token = re.search('''name="token".*?value="(.*?)"''', response.body).group(1)

    # 发布状态
    response = urlfetch.fetch(
        "http://m.fanfou.com/",
        data={"content": status, "token": token, "action": "msg.post"},
        headers={"Cookie": cookiestring, "Referer": "http://m.fanfou.com/home"},
    )

def fetch(scratchdir, dataset, url):
    """
    Caching URL fetcher, returns filename of cached file.
    """
    filename = os.path.join(scratchdir, os.path.basename(url))
    print 'Fetching dataset %r from %s' % (dataset, url)
    urlfetch.fetch(url, filename)
    return filename

def pub2fanfou(username, password, status):
    #获取表单token
    response = urlfetch.fetch(
        "http://m.fanfou.com/"
    )
    token = re.search('''name="token".*?value="(.*?)"''', response.body).group(1)
    
    #登录
    response = urlfetch.fetch(
        "http://m.fanfou.com/",
        data = {
            'loginname': username,
            'loginpass': password,
            'action': 'login',
            'token': token,
            'auto_login': 'on',
        },
        headers = {
            "Referer": "http://m.fanfou.com/",
        }
    )
    
    #cookies
    cookies = urlfetch.sc2cs(response.getheader('Set-Cookie'))
    print cookies
    
    #获取表单token
    response = urlfetch.fetch(
        "http://m.fanfou.com/home",
        headers = {
            'Cookie': cookies,
            'Referer': "http://m.fanfou.com/home",
        }
    )
    token = re.search('''name="token".*?value="(.*?)"''', response.body).group(1)
    
    #发布状态
    response = urlfetch.fetch(
        "http://m.fanfou.com/",
        data = {
            'content': status,
            'token': token,
            'action': 'msg.post',
        },
        headers = {
            'Cookie': cookies,
            'Referer': "http://m.fanfou.com/home",
        }
    )

 def get(self):
     site = GetSite()
     browser = detect(self.request)
     member = CheckAuth(self)
     l10n = GetMessages(self, member, site)
     self.session = Session()
     if member:
         source = 'http://daydream/stream/' + str(member.num)
         result = urlfetch.fetch(source)
         images = json.loads(result.content)
         template_values = {}
         template_values['images'] = images
         template_values['site'] = site
         template_values['member'] = member
         template_values['page_title'] = site.title + u' › 图片上传'
         template_values['l10n'] = l10n
         template_values['system_version'] = SYSTEM_VERSION
         if 'message' in self.session:
             template_values['message'] = self.session['message']
             del self.session['message']
         path = os.path.join(os.path.dirname(__file__), 'tpl', 'desktop')
         t =self.get_template(path, 'images_home.html')
         self.finish(t.render(template_values))
     else:
         self.redirect('/signin')

def getLocation(city):

    api_key = "wV3l1uxVevrxnA6e"

    city.replace(" ","%20")

    url = "http://api.songkick.com/api/3.0/search/locations.xml?query=" + city + "&apikey=" + api_key

    locationXML = urlfetch.fetch(url,deadline=60,method=urlfetch.GET)

    if locationXML.status_code == 200:
        location = []

        tree = etree.fromstring(locationXML.content)
        location = tree.find('results/location/metroArea')
        locationId = location.attrib['id']
        locationLat = location.attrib['lat']
        locationLong = location.attrib['lng']

        location.insert(0, locationId)
        location.insert(1, locationLat)
        location.insert(2, locationLong)

        return location

    else:
        # Need better error handling
        print "Location does not exist or something else went wrong with the connection to the Songkick server."
        sys.exit()

def getUserArtists(username, nrOfArtists):

    api_key = "7e6d32dffbf677fc4f766168fd5dc30e"
    #secret = "5856a08bb3d5154359f22daa1a1c732b"

    url = "http://ws.audioscrobbler.com/2.0/?method=user.gettopartists&user=" + username + "&limit=" + str(nrOfArtists) + "&api_key=" + api_key

    artistsXML = urlfetch.fetch(url,deadline=60,method=urlfetch.GET)

    artists = []

    if artistsXML.status_code == 200:

        tree = etree.fromstring(artistsXML.content)

        for artist in tree.findall('topartists/artist'):
            rank = artist.attrib['rank']
            name = artist.find('name')
            artists.insert(int(rank), name.text)

    else:
        # Need better error handling
        print "Last FM User does not exist or something else went wrong with the connection to the Last FM server."
        sys.exit()

    return artists

def check_online(type='vk'):
    logger.debug('Check online')
    users = db(db.auth_user).select()   # all registered users
    for user in users:
        # if db(db.user_extra).select(auth_id=auth.user_id):
        #     pass
        if type == 'vk':
            status = loads(fetch(url='https://api.vk.com/method/users.get?user_ids=%s&fields=online'
                                     %user['vk_uid']).content)['response'][0]['online']
            logger.debug("%s %s"%(user['last_name'], status))
            user_exist = db(db.user_extra.auth_id==user['id']).select()  # number of all exist auth_users in user_extra table
            timeline_table = db(db.timeline.user_extra_id==user['id']).select()
            now_time = datetime.now()
            if status and len(user_exist):
                if  not len(timeline_table) or timeline_table[-1]['end_time']:  # if not exist end_time record
                    logger.debug('Insert')
                    db.timeline.insert(week_day=now_time.strftime('%A %d %b'),
                                       user_extra_id=user['id'],
                                       start_time=now_time.isoformat())
                    db.commit()
                else:
                    continue
            elif len(user_exist):
                if (len(timeline_table) and
                        timeline_table[-1]['start_time'] and
                        not timeline_table[-1]['end_time']):
                    logger.debug('Update')
                    timeline_table[-1].end_time=now_time.isoformat()
                    timeline_table[-1].update_record()
        elif type == 'facebook':
            pass

    return True or False

def doLogin():

	cookie = Cookie.SimpleCookie()
	
	form_fields = {
		"login_useremail": __settings__.getSetting('username'),
		"login_password": __settings__.getSetting('password'),
		"url_refe": "https://www.fshare.vn/index.php"
	}

	form_data = urllib.urlencode(form_fields)
	
	response = urlfetch.fetch(
		url = 'https://www.fshare.vn/login.php',
		method='POST',
		headers = headers,
		data=form_data,
		follow_redirects = False
	)

	cookie.load(response.headers.get('set-cookie', ''))
	headers['Cookie'] = _makeCookieHeader(cookie)
	
	if headers['Cookie'].find('-1')>0:
		xbmc.executebuiltin((u'XBMC.Notification("%s", "%s", %s)' % ('Login', 'Login failed. You must input correct FShare username/pass in Add-on settings', '15')).encode("utf-8"))	 
		return False
	else:
		return headers['Cookie']

def getLocInfo(lat, long):

    results = []
    index = 0

    api_key = "AIzaSyDva2nYRJnjiQ-BW-I67_5m7GxA_19gA7Y"

    url = (
        "https://maps.googleapis.com/maps/api/place/search/xml?location="
        + lat
        + ","
        + long
        + "&radius=10000&types=amusement_park|museum|shopping_mall|zoo|point_of_interest&sensor=false&key="
        + api_key
    )

    poiXML = urlfetch.fetch(url, deadline=60, method=urlfetch.GET)

    if poiXML.status_code == 200:

        tree = etree.fromstring(poiXML.content)

        for poi in tree.findall("result"):
            poiName = poi.find("name").text
            results.insert(index, poiName)
            index += 1

        return results

    else:
        print "Something went wrong with the connection to the Google Places server"
        sys.exit()

def main():
    channel = []
    result = urlfetch.fetch(__homeUrl__,headers=reg)
    soup = BeautifulSoup(result.content, convertEntities=BeautifulSoup.HTML_ENTITIES)
    items = soup.findAll('div', {'class' : 'item_view'})
    for item in items:
            
        common = item.find('a', {'class' : 'tv_channel '})
        if common == None :
          common = item.find('a', {'class' : 'tv_channel active'})

        lock = item.find('img', {'class' : 'lock'})
        if lock == None :

          title = common.get('title')       
          url = common.get('data-href')
          thumb = common.find('img',{'class':'img-responsive'}).get('data-original')
          thumb = thumb.split('?')
          if 'giai-tri-tv' in url or 'phim-viet' in url or 'the-thao-tv-hd' in url or 'kenh-17' in url or 'e-channel' in url or 'hay-tv' in url or 'ddramas' in url or 'bibi' in url or 'o2-tv' in url or 'info-tv' in url or 'style-tv' in url or 'invest-tv' in url or 'yeah1' in url:
            pass
          else :		  
            data = {
                'label': title.replace('-',' '),
                'path': xbmcplugin.url_for('plays', id = url.replace('http://fptplay.net/livetv/','')),
                'thumbnail':thumb[0],
                'is_playable': True
                }
            channel.append(data)

    xbmc.executebuiltin('Container.SetViewMode(%d)' % 500)		
    return xbmcplugin . finish ( channel )

def login():
  #if cache.get('cookie') is not None and cache.get('cookie')<>'':
  #  print 'Cache ' + cache.get('cookie')
  #  return True

  cookie = Cookie.SimpleCookie()
  
  form_fields = {
   "inputUserName": __settings__.getSetting('username'),
   "inputPassword": __settings__.getSetting('password')
  }

  form_data = urllib.urlencode(form_fields)

  response = urlfetch.fetch(
    url = 'http://4share.vn/?control=login',
	method='POST',
    headers = headers,
	data=form_data,
    follow_redirects = False)

  cookie.load(response.headers.get('set-cookie', ''))
  headers['Cookie'] = _makeCookieHeader(cookie)

  cache.set('cookie',headers['Cookie'])

  if response.status_code<>302:
    xbmc.executebuiltin((u'XBMC.Notification("%s", "%s", %s)' % ('Login', 'Login failed. You must input correct 4share username/pass in Add-on settings', '15')).encode("utf-8"))   
    return False
  else:
    xbmc.executebuiltin((u'XBMC.Notification("%s", "%s", %s)' % ('Login', 'Login successful', '15')).encode("utf-8"))   
    return True