Python urllib2.unquote函数代码示例

def convert_utf8_url_to_ascii(url):
    """
    taken from http://stackoverflow.com/questions/804336/best-way-to-convert-a-unicode-url-to-ascii-utf-8-percent-escaped-in-python
    """
       # turn string into unicode
    if not isinstance(url, unicode):
        url = url.decode('utf8')

    # parse it
    parsed = urlparse.urlsplit(url)

    # divide the netloc further
    userpass, at, hostport = parsed.netloc.rpartition('@')
    user, colon1, pass_ = userpass.partition(':')
    host, colon2, port = hostport.partition(':')

    # encode each component
    scheme = parsed.scheme.encode('utf8')
    user = urllib2.quote(user.encode('utf8'))
    colon1 = colon1.encode('utf8')
    pass_ = urllib2.quote(pass_.encode('utf8'))
    at = at.encode('utf8')
    host = host.encode('idna')
    colon2 = colon2.encode('utf8')
    port = port.encode('utf8')
    # could be encoded slashes!
    path = '/'.join(urllib2.quote(urllib2.unquote(pce).encode('utf8'), '')
                    for pce in parsed.path.split('/'))
    query = urllib2.quote(urllib2.unquote(parsed.query).encode('utf8'), '=&?/')
    fragment = urllib2.quote(urllib2.unquote(parsed.fragment).encode('utf8'))

    # put it back together
    netloc = ''.join((user, colon1, pass_, at, host, colon2, port))
    return urlparse.urlunsplit((scheme, netloc, path, query, fragment))

 def set_language(self):
     "Set the language"
     nextpage = request.params.get('next', None)
     if not nextpage:
         nextpage = request.headers.get('Referer', None)
     if not nextpage:
         nextpage = '/'
     if '://' in nextpage:
         from_url = urlparse(nextpage)
         nextpage = from_url[2]
     lang_code = request.params.get('language', None)
     if lang_code and check_language(lang_code):
         session['lang'] = lang_code
         session.save()
     params = []
     for param in request.params:
         if not param in ['language', 'amp']:
             value = request.params[param]
             if value:
                 if (param == 'came_from' and
                     '://' in urllib2.unquote(value)):
                     urlparts = urlparse(urllib2.unquote(value))
                     value = urlparts[2] or '/'
                 params.append('%s=%s' % (urllib2.quote(param),
                                         urllib2.quote(value)))
     if 'lc=1' not in params:
         params.append('lc=1')
     if params:
         nextpage = "%s?%s" % (nextpage, '&'.join(params))
     redirect(nextpage)

    def resolve_url(self, url, connection, arguments={}):
        handler = None
        args = []
        kwargs = {}

        # unable to check host at the moment, so just loop over all handlers
        for pattern, handlers in self.application.handlers:
            for spec in handlers:
                if issubclass(spec.handler_class, AiryHandler):
                    match = spec.regex.match(url)
                    if match:
                        if spec.regex.groups:
                            # None-safe wrapper around url_unescape to handle
                            # unmatched optional groups correctly
                            def unquote(s):
                                if s is None:
                                    return s
                                return escape.url_unescape(s, encoding=None)
                                # Pass matched groups to the handler.  Since

                            # match.groups() includes both named and unnamed groups,
                            # we want to use either groups or groupdict but not both.
                            # Note that args are passed as bytes so the handler can
                            # decide what encoding to use.

                            if spec.regex.groupindex:
                                kwargs = dict((k, unquote(v)) for (k, v) in match.groupdict().iteritems())
                            else:
                                args = [unquote(s) for s in match.groups()]
                        handler = spec.handler_class(self, connection, arguments, **spec.kwargs)
                        break

        return handler, args, kwargs

def main():
    if len(sys.argv) == 1:
        dom = parseString(urlopen(XML_URL, data=POST_BODY).read())
    elif len(sys.argv) == 2:
        filename = sys.argv[1]
        if not os.path.exists(filename):
            print "File not found"
            return 1
        dom = parse(filename)
    else:
        print "Invalid arguments."
        return 1
    
    data = unquote(dom.getElementsByTagName("data").pop().lastChild.toxml()).split(':')[1].split(',')
    
    for i in range(0, len(data), len(FIELDS)):
        t_fields = data[i:i+len(FIELDS)]
        d = dict(zip(FIELDS, t_fields))
        d['url'] = unquote(d['url'])
        d['flv_url'] = unquote(d['flv_url'])
        d['rtmp_url'] = unquote(d['rtmp_url'])
        
        print d['title'].replace('+', ' ')
        print d['url']
        print d['rtmp_url']
        print    

 def post(self):
   username = self.request.get('username')
   lua_script = urllib2.unquote(base64.b64decode(
       self.request.get('lua_script')))
   lua_aggregator = urllib2.unquote(base64.b64decode(
       self.request.get('lua_aggregator')))
   _AddLuaTask(self.request, username, lua_script, lua_aggregator)

def grab(secondary_url):
    global DEPTH
    global LIMIT_DEPTH

    if (secondary_url, DEPTH) in URLS:
        return True

    URLS.append((secondary_url, DEPTH))

    filename = urllib2.unquote(secondary_url.split('/')[-1])
    filename, isimg = fix_image(filename)

    if exist(filename):
        return True

    if isimg:
        get_wiki_image(secondary_url)
        return True

    if DEPTH > LIMIT_DEPTH:
        return True

    print '[%d]' % DEPTH, '--' * DEPTH, 'parsing...\t', urllib2.unquote(secondary_url)
    DEPTH += 1

    try:
        first_page = urllib2.urlopen(URL_BASE + secondary_url)
    except Exception, inst:
        print URL_BASE + urllib2.unquote(secondary_url), 'no existe'
        return False

    def http_get(self, url):
            MIME = '*/*'
            unquoteurl = urllib2.unquote(url.encode('utf-8'))
            scheme,netloc,url,params,query,fragment = urlparse(unquoteurl)
	    netloc=urllib2.quote(netloc)
            url = urllib2.quote(url)
            url = ParseResult( scheme,netloc,url,params,query,fragment ).geturl()
            retries = 30
            i = 0
            while(True):
                try:
                    if(self.useproxy):
			print 'using proxy'
                        response = self.opener.open(url,timeout=5)
                        print("GET "  + urllib2.unquote(response.geturl().encode()) + " " + str(response.code))
                        if('content-type' in response.headers):
                            MIME = response.headers['content-type'].split(';')[0]
			print response
                        return response.read(), response.code, MIME
                    else:
                        response = requests.get(url)
                        print("GET "  + urllib2.unquote(str(response.url)) + " " + str(response.status_code))
                        if('content-type' in response.headers):
                            MIME = response.headers['content-type'].split(';')[0]
                        return response.content, response.status_code, MIME
                except:
                    if(i > retries):
                        print traceback.print_exc()
                        raise sys.exc_info()[0]
                    print "timeout 5000ms"
                    i += 1

def loadVideos(url,name):
        #try:
           newlink=url
           print newlink
           xbmc.executebuiltin("XBMC.Notification(Please Wait!,Loading selected video)")
           if (newlink.find("dailymotion") > -1):
                match=re.compile('(dailymotion\.com\/(watch\?(.*&)?v=|(embed|v|user)\/))([^\?&"\'>]+)').findall(newlink)
                lastmatch = match[0][len(match[0])-1]
                link = 'http://www.dailymotion.com/'+str(lastmatch)
                req = urllib2.Request(link)
                req.add_header('User-Agent', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-GB; rv:1.9.0.3) Gecko/2008092417 Firefox/3.0.3')
                response = urllib2.urlopen(req)
                link=response.read()
                response.close()
                sequence=re.compile('"sequence",  "(.+?)"').findall(link)
                newseqeunce = urllib.unquote(sequence[0]).decode('utf8').replace('\\/','/')
                #print 'in dailymontion:' + str(newseqeunce)
                imgSrc=re.compile('"videoPreviewURL":"(.+?)"').findall(newseqeunce)
                if(len(imgSrc[0]) == 0):
                	imgSrc=re.compile('/jpeg" href="(.+?)"').findall(link)
                dm_low=re.compile('"sdURL":"(.+?)"').findall(newseqeunce)
                dm_high=re.compile('"hqURL":"(.+?)"').findall(newseqeunce)
                playVideo('dailymontion',urllib2.unquote(dm_low[0]).decode("utf8"))
           elif (newlink.find("docs.google.com") > -1):  
                vidcontent = GetContent(newlink)
                html = vidcontent.decode('utf8')
                stream_map = re.compile('fmt_stream_map","(.+?)"').findall(html)[0].replace("\/", "/")
                formatArray = stream_map.split(',')
                for formatContent in formatArray:
                     formatContentInfo = formatContent.split('|')
                     qual = formatContentInfo[0]
                     url = (formatContentInfo[1]).decode('unicode-escape')
                     playVideo("direct",url)
           elif (newlink.find("4shared") > -1):
                d = xbmcgui.Dialog()
                d.ok('Not Implemented','Sorry 4Shared links',' not implemented yet')		
           elif (newlink.find("vimeo") > -1):
                idmatch =re.compile("http://player.vimeo.com/video/([^\?&\"\'>]+)").findall(newlink)
                if(len(idmatch) > 0):
                        idmatch=idmatch[0].replace("'","")
                else:
                        idmatch =re.compile("//vimeo.com/(.+?)dk").findall(newlink+"dk")
                        idmatch=idmatch[0].replace("'","")
                playVideo('vimeo',idmatch)
           else:
                if (newlink.find("linksend.net") > -1):
                     d = xbmcgui.Dialog()
                     d.ok('Not Implemented','Sorry videos on linksend.net does not work','Site seem to not exist')		
                newlink1 = urllib2.unquote(newlink).decode("utf8")+'&dk;'
                print 'NEW url = '+ newlink1
                match=re.compile('(youtu\.be\/|youtube-nocookie\.com\/|youtube\.com\/(watch\?(.*&)?v=|(embed|v|user)\/))([^\?&"\'>]+)').findall(newlink1)
                if(len(match) == 0):
                    match=re.compile('http://www.youtube.com/watch\?v=(.+?)&dk;').findall(newlink1)
                if(len(match) > 0):
                    lastmatch = match[0][len(match[0])-1].replace('v/','')
                    #d = xbmcgui.Dialog()
                    #d.ok('mode 2',str(lastmatch),'launching yout')
                    playVideo('youtube',lastmatch)
                else:
                    playVideo('moviekhmer',urllib2.unquote(newlink).decode("utf8"))

    def test_with_one_batch_open(self):
        self.batch.open_for_location(self.location)

        self.assertEquals(HouseholdMemberBatchCompletion.objects.count(), 0)
        mock_filter = MagicMock()
        mock_filter.exists.return_value = True
        with patch.object(RandomHouseHoldSelection.objects, 'filter', return_value=mock_filter):
            with patch.object(USSDSurvey, 'is_active', return_value=False):
                self.reset_session()

                self.choose_menu_to_take_survey()
                self.select_household()
                response = self.select_household_member()
                response_string = "responseString=%s&action=request" % self.question_1.to_ussd()
                self.assertEquals(urllib2.unquote(response.content), response_string)

                response = self.respond("1")
                response_string = "responseString=%s&action=request" % self.question_2.to_ussd()
                self.assertEquals(urllib2.unquote(response.content), response_string)

                response = self.respond("1")
                response_string = "responseString=%s&action=request" % USSD.MESSAGES['HOUSEHOLD_COMPLETION_MESSAGE']
                self.assertEquals(urllib2.unquote(response.content), response_string)

                self.assertEquals(HouseholdMemberBatchCompletion.objects.count(), 1)
                household_completed = HouseholdMemberBatchCompletion.objects.latest('id')
                self.assertEquals(household_completed.household, self.household)
                self.assertEquals(household_completed.investigator, self.investigator)
                self.assertEquals(household_completed.batch, self.batch)

    def test_selection_for_survey_that_has_no_sampling(self):
        open_survey = Survey.objects.create(name="open survey", description="open survey", has_sampling=False)
        with patch.object(Survey, "currently_open_survey", return_value=open_survey):
            mobile_number = self.ussd_params['msisdn'].replace(COUNTRY_PHONE_CODE, '', 1)
            self.assertEquals(RandomHouseHoldSelection.objects.count(), 0)

            response_message = "responseString=%s&action=request" % HouseHoldSelection.MESSAGES['HOUSEHOLDS_COUNT_QUESTION']

            response = self.client.get('/ussd', data=self.ussd_params)
            self.failUnlessEqual(response.status_code, 200)
            self.assertEquals(urllib2.unquote(response.content), response_message)
            self.assertEquals(RandomHouseHoldSelection.objects.count(), 0)

            self.ussd_params['response'] = "true"
            self.ussd_params['ussdRequestString'] = " 100 "

            response_message = "responseString=%s&action=end" % HouseHoldSelection.MESSAGES[
                'HOUSEHOLD_CONFIRMATION_MESSAGE']
            response = self.client.get('/ussd', data=self.ussd_params)
            self.failUnlessEqual(response.status_code, 200)
            self.assertEquals(urllib2.unquote(response.content), response_message)
            self.assertEquals(RandomHouseHoldSelection.objects.count(), 1)

            household_selection = RandomHouseHoldSelection.objects.all()[0]
            self.assertEquals(household_selection.mobile_number, mobile_number)
            self.assertEquals(household_selection.no_of_households, 100)
            selected_households = household_selection.selected_households.split(',')
            self.assertEquals(len(selected_households), 100)

            message = BackendMessage.objects.filter(identity=self.ussd_params['msisdn'])
            self.failIf(message)

def _get_video_link_dict(url):
	header = {'GData-Version' : '3.0' }
	req = urllib2.Request(url, None, header)
	try:
		response = urllib2.urlopen(req)
	except urllib2.URLError as e:
		if hasattr(e, 'reason'):
			raise RuntimeError(str(e.reason))
		elif hasattr(e, 'code'):
			raise RuntimeError(str(e.code))
	else:
		response_data = response.read()

	re_stream_map = re.compile(r'"url_encoded_fmt_stream_map": "(.+?)"')
	re_adaptive_fmts = re.compile(r'"adaptive_fmts": "(.+?)"')
	re_url = re.compile(r'url=(.+?)(?:,|\\)')
	re_itag = re.compile(r'itag=(\d+)')

	stream_map = re.search(re_stream_map, response_data).group(1)
	adaptive_fmts = re.search(re_adaptive_fmts, response_data).group(1)

	video_info = stream_map + adaptive_fmts
	
	urls = re.findall(re_url, video_info)

	url_dict = {}

	for u in urls:
		u = urllib2.unquote(urllib2.unquote(u))
		itag = re.search(re_itag, u).group(1)
		url_dict[str(itag)] = u
	
	return url_dict

    def test_selection(self):
        with patch.object(Survey, "currently_open_survey", return_value=self.open_survey):
            self.assertEquals(RandomHouseHoldSelection.objects.count(), 0)

            response_message = "responseString=%s&action=request" % HouseHoldSelection.MESSAGES['HOUSEHOLDS_COUNT_QUESTION']
            response = self.client.get('/ussd', data=self.ussd_params)
            self.failUnlessEqual(response.status_code, 200)
            self.assertEquals(urllib2.unquote(response.content), response_message)
            self.assertEquals(RandomHouseHoldSelection.objects.count(), 0)

            self.ussd_params['response'] = "true"
            self.ussd_params['ussdRequestString'] = " 100 "

            response_message = "responseString=%s&action=end" % HouseHoldSelection.MESSAGES[
                'HOUSEHOLD_SELECTION_SMS_MESSAGE']
            response = self.client.get('/ussd', data=self.ussd_params)
            self.failUnlessEqual(response.status_code, 200)
            self.assertEquals(urllib2.unquote(response.content), response_message)
            self.assertEquals(RandomHouseHoldSelection.objects.count(), 1)

            household_selection = RandomHouseHoldSelection.objects.all()[0]
            mobile_number = self.ussd_params['msisdn'].replace(COUNTRY_PHONE_CODE, '', 1)
            self.assertEquals(household_selection.mobile_number, mobile_number)
            self.assertEquals(household_selection.no_of_households, 100)
            selected_households = household_selection.selected_households.split(',')
            self.assertEquals(len(selected_households), 10)

            message = BackendMessage.objects.get(identity=self.ussd_params['msisdn'])
            self.assertEquals(message.text, household_selection.text_message())

 def changepassword(self, fields):
     time.sleep(1)
     d={urllib2.unquote(i.split('=')[0]):urllib2.unquote(i.split('=')[1]) for i in [tmp for tmp in fields.split('&')]}
     oldpassword=d['oldpassword']
     newpassword1=d['newpassword1']
     newpassword2=d['newpassword2']
     user=checkPassword()
     if not user:
         return """You have been logged out.  <a href="/home/login">Log in</a>."""
     if oldpassword=='' or newpassword1=='' or newpassword2=='':
         return 'You need to fill in all the required password data'
     if newpassword1!=newpassword2:
         return "You entered your new password incorrectly.  Please try again."
     elif len(newpassword1)<=5:
         return "<p>Your password needs to be greater than 5 characters</p>" 
     oldpass=hashlib.md5((oldpassword+salt).encode('utf-8')).hexdigest()
     newpass=hashlib.md5((newpassword1+salt).encode('utf-8')).hexdigest()
     ## RESET COOKIE
     cookie = cherrypy.response.cookie
     sida=user
     sidb=newpass
     cookie['sida']=sida
     cookie['sida']['expires'] = 12 * 30 * 24 * 60 * 60
     cookie['sida']["path"] = "/"
     cookie['sidb']=sidb
     cookie['sidb']['expires'] = 12 * 30 * 24 * 60 * 60
     cookie['sidb']["path"] = "/"
     cherrypy.request.cookie=cookie
     
     data=db.execute("""SELECT id FROM lab_members WHERE name=%s AND password=%s""", (user,oldpass), commit=False)
     if len(data)==0:
         return "Your password is incorrect."
     db.execute('''UPDATE lab_members SET  password=%s WHERE name=%s''', (newpass, user))
     return 'Your password has been updated!'

def lambe(orgao_a, estado_a, orgao_b, estado_b, raw=False):
    orgao_a = unquote(orgao_a)
    estado_a = unquote(estado_a)
    orgao_b = unquote(orgao_b)
    estado_b = unquote(estado_b)

    l = {
        "a": {"estado": estado_a, "orgao": orgao_a, "valor": lambes[orgao_a][estado_a]},
        "b": {"estado": estado_b, "orgao": orgao_b, "valor": lambes[orgao_b][estado_b]},
    }

    if l["a"]["valor"] <= l["b"]["valor"]:
        l["proporcao"] = "menos"
        # hackish?
        l["razao"] = round((l["b"]["valor"] - l["a"]["valor"]) / l["a"]["valor"], 1)
        l["razao_g"] = round(l["b"]["valor"] / l["a"]["valor"], 1)
    else:
        l["proporcao"] = "mais"
        l["razao"] = round(l["a"]["valor"] / l["b"]["valor"], 1)
    image_path = unidecode("-".join([l["a"]["orgao"], l["a"]["estado"], l["b"]["orgao"], l["b"]["estado"]]))

    if os.path.isfile(here + "/static/raw/" + image_path + "-hi.png"):
        l["image"] = image_path
    else:
        if not raw:
            t = threading.Thread(target=generate_image, args=(l, 3))
            t.start()
    return render_template("lambe.html", l=l)

def getImage(url, h=None, w=None, o=100, auth=None, **kwargs):
    # Create image directory if it doesnt exist
    imgdir = os.path.join(htpc.datadir, 'images/')
    if not os.path.exists(imgdir):
        os.makedirs(imgdir)

    fileName = unquote(unquote(url)).rsplit('/', 1).pop()
    imgName, imgType = fileName.rsplit('.', 1)

    original = resized = os.path.join(imgdir, imgType+'_'+imgName+'.png')
    if h and w:
        resized = os.path.join(imgdir, original+'_'+w+'_'+h+'.png')

    # If there is no local copy of the original
    if not os.path.isfile(original):
        original = downloadImage(url, original, auth)

    if not original:
        print "Error downloading image"
        raise cherrypy.NotFound(fileName)

    # If there is no local resized copy
    if not os.path.isfile(resized):
        resized = resizeImage(original, h, w, o, resized)

    if not resized:
        resized = original

    # Load file from disk
    return serve_file(path=resized, content_type='image/png')