本文整理匯總了Python中pycurl.EFFECTIVE_URL屬性的典型用法代碼示例。如果您正苦於以下問題:Python pycurl.EFFECTIVE_URL屬性的具體用法?Python pycurl.EFFECTIVE_URL怎麽用?Python pycurl.EFFECTIVE_URL使用的例子?那麽, 這裏精選的屬性代碼示例或許可以為您提供幫助。您也可以進一步了解該屬性所在類pycurl
的用法示例。
在下文中一共展示了pycurl.EFFECTIVE_URL屬性的15個代碼示例,這些例子默認根據受歡迎程度排序。您可以為喜歡或者感覺有用的代碼點讚,您的評價將有助於係統推薦出更棒的Python代碼示例。
示例1: fetch
# 需要導入模塊: import pycurl [as 別名]
# 或者: from pycurl import EFFECTIVE_URL [as 別名]
def fetch(self, request, **kwargs):
"""Executes an HTTPRequest, returning an HTTPResponse.
If an error occurs during the fetch, we raise an HTTPError.
"""
if not isinstance(request, HTTPRequest):
request = HTTPRequest(url=request, **kwargs)
buffer = cStringIO.StringIO()
headers = httputil.HTTPHeaders()
try:
_curl_setup_request(self._curl, request, buffer, headers)
self._curl.perform()
code = self._curl.getinfo(pycurl.HTTP_CODE)
effective_url = self._curl.getinfo(pycurl.EFFECTIVE_URL)
buffer.seek(0)
response = HTTPResponse(
request=request, code=code, headers=headers,
buffer=buffer, effective_url=effective_url)
if code < 200 or code >= 300:
raise HTTPError(code, response=response)
return response
except pycurl.error, e:
buffer.close()
raise CurlError(*e)
示例2: _finish
# 需要導入模塊: import pycurl [as 別名]
# 或者: from pycurl import EFFECTIVE_URL [as 別名]
def _finish(self, curl, curl_error=None, curl_message=None):
info = curl.info
curl.info = None
self._multi.remove_handle(curl)
self._free_list.append(curl)
buffer = info["buffer"]
if curl_error:
error = CurlError(curl_error, curl_message)
code = error.code
effective_url = None
buffer.close()
buffer = None
else:
error = None
code = curl.getinfo(pycurl.HTTP_CODE)
effective_url = curl.getinfo(pycurl.EFFECTIVE_URL)
buffer.seek(0)
# the various curl timings are documented at
# http://curl.haxx.se/libcurl/c/curl_easy_getinfo.html
time_info = dict(
queue=info["curl_start_time"] - info["request"].start_time,
namelookup=curl.getinfo(pycurl.NAMELOOKUP_TIME),
connect=curl.getinfo(pycurl.CONNECT_TIME),
pretransfer=curl.getinfo(pycurl.PRETRANSFER_TIME),
starttransfer=curl.getinfo(pycurl.STARTTRANSFER_TIME),
total=curl.getinfo(pycurl.TOTAL_TIME),
redirect=curl.getinfo(pycurl.REDIRECT_TIME),
)
try:
info["callback"](HTTPResponse(
request=info["request"], code=code, headers=info["headers"],
buffer=buffer, effective_url=effective_url, error=error,
reason=info['headers'].get("X-Http-Reason", None),
request_time=time.time() - info["curl_start_time"],
time_info=time_info))
except Exception:
self.handle_callback_exception(info["callback"])
示例3: _move_path
# 需要導入模塊: import pycurl [as 別名]
# 或者: from pycurl import EFFECTIVE_URL [as 別名]
def _move_path(self):
"""
Move the downloaded file to the authentic path (identified by
effective URL)
"""
if is_temp_path(self._path) and self._pycurl is not None:
eurl = self._pycurl.getinfo(pycurl.EFFECTIVE_URL)
er = get_resource_name(eurl)
r = get_resource_name(self.url)
if er != r and os.path.exists(self.path):
new_path = self._get_path(self._path, eurl)
shutil.move(self.path, new_path)
self.path = new_path
示例4: _finish
# 需要導入模塊: import pycurl [as 別名]
# 或者: from pycurl import EFFECTIVE_URL [as 別名]
def _finish(self, curl, curl_error=None, curl_message=None):
info = curl.info
curl.info = None
self._multi.remove_handle(curl)
self._free_list.append(curl)
buffer = info["buffer"]
if curl_error:
error = CurlError(curl_error, curl_message)
code = error.code
effective_url = None
buffer.close()
buffer = None
else:
error = None
code = curl.getinfo(pycurl.HTTP_CODE)
effective_url = curl.getinfo(pycurl.EFFECTIVE_URL)
buffer.seek(0)
# the various curl timings are documented at
# http://curl.haxx.se/libcurl/c/curl_easy_getinfo.html
time_info = dict(
queue=info["curl_start_time"] - info["request"].start_time,
namelookup=curl.getinfo(pycurl.NAMELOOKUP_TIME),
connect=curl.getinfo(pycurl.CONNECT_TIME),
pretransfer=curl.getinfo(pycurl.PRETRANSFER_TIME),
starttransfer=curl.getinfo(pycurl.STARTTRANSFER_TIME),
total=curl.getinfo(pycurl.TOTAL_TIME),
redirect=curl.getinfo(pycurl.REDIRECT_TIME),
)
try:
info["callback"](HTTPResponse(
request=info["request"], code=code, headers=info["headers"],
buffer=buffer, effective_url=effective_url, error=error,
request_time=time.time() - info["curl_start_time"],
time_info=time_info))
except Exception:
self.handle_callback_exception(info["callback"])
示例5: url_check
# 需要導入模塊: import pycurl [as 別名]
# 或者: from pycurl import EFFECTIVE_URL [as 別名]
def url_check(self, url):
'''下載地址檢查'''
url_info = {}
proto = urlparse.urlparse(url)[0]
if proto not in VALIDPROTOCOL:
print 'Valid protocol should be http or ftp, but % s found < %s >!' % (proto, url)
else:
ss = StringIO()
curl = pycurl.Curl()
curl.setopt(pycurl.FOLLOWLOCATION, 1)
curl.setopt(pycurl.MAXREDIRS, 5)
curl.setopt(pycurl.CONNECTTIMEOUT, 30)
curl.setopt(pycurl.TIMEOUT, 300)
curl.setopt(pycurl.NOSIGNAL, 1)
curl.setopt(pycurl.NOPROGRESS, 1)
curl.setopt(pycurl.NOBODY, 1)
curl.setopt(pycurl.HEADERFUNCTION, ss.write)
curl.setopt(pycurl.URL, url)
try:
curl.perform()
except:
pass
if curl.errstr() == '' and curl.getinfo(pycurl.RESPONSE_CODE) in STATUS_OK:
url_info['url'] = curl.getinfo(pycurl.EFFECTIVE_URL)
url_info['file'] = os.path.split(url_info['url'])[1]
url_info['size'] = int(
curl.getinfo(pycurl.CONTENT_LENGTH_DOWNLOAD))
url_info['partible'] = (ss.getvalue().find('Accept - Ranges') != -1)
return url_info
示例6: finalize
# 需要導入模塊: import pycurl [as 別名]
# 或者: from pycurl import EFFECTIVE_URL [as 別名]
def finalize(self, c):
"""finalize a Curl object and extract information from it."""
self._url = c.getinfo(pycurl.EFFECTIVE_URL)
# timing info
nt = c.getinfo(pycurl.NAMELOOKUP_TIME)
ct = c.getinfo(pycurl.CONNECT_TIME)
pt = c.getinfo(pycurl.PRETRANSFER_TIME)
st = c.getinfo(pycurl.STARTTRANSFER_TIME)
tt = c.getinfo(pycurl.TOTAL_TIME)
rt = c.getinfo(pycurl.REDIRECT_TIME)
self._timing = TimingInfo(nt, ct, pt, st, tt, rt)
self._redirectcount = c.getinfo(pycurl.REDIRECT_COUNT)
self._cookielist = c.getinfo(pycurl.INFO_COOKIELIST)
c.close()
self._downloadfile = None
示例7: _finish
# 需要導入模塊: import pycurl [as 別名]
# 或者: from pycurl import EFFECTIVE_URL [as 別名]
def _finish(self, curl, curl_error=None, curl_message=None):
info = curl.info
curl.info = None
self._multi.remove_handle(curl)
self._free_list.append(curl)
buffer = info["buffer"]
if curl_error:
error = CurlError(curl_error, curl_message)
code = error.code
effective_url = None
buffer.close()
buffer = None
else:
error = None
code = curl.getinfo(pycurl.HTTP_CODE)
effective_url = curl.getinfo(pycurl.EFFECTIVE_URL)
buffer.seek(0)
# the various curl timings are documented at
# http://curl.haxx.se/libcurl/c/curl_easy_getinfo.html
time_info = dict(
queue=info["curl_start_time"] - info["request"].start_time,
namelookup=curl.getinfo(pycurl.NAMELOOKUP_TIME),
connect=curl.getinfo(pycurl.CONNECT_TIME),
pretransfer=curl.getinfo(pycurl.PRETRANSFER_TIME),
starttransfer=curl.getinfo(pycurl.STARTTRANSFER_TIME),
total=curl.getinfo(pycurl.TOTAL_TIME),
redirect=curl.getinfo(pycurl.REDIRECT_TIME),
)
try:
info["callback"](HTTPResponse(
request=info["request"], code=code, headers=info["headers"],
buffer=buffer, effective_url=effective_url, error=error,
request_time=monotime() - info["curl_start_time"],
time_info=time_info))
except Exception:
self.handle_callback_exception(info["callback"])
示例8: _finish
# 需要導入模塊: import pycurl [as 別名]
# 或者: from pycurl import EFFECTIVE_URL [as 別名]
def _finish(self, curl, curl_error=None, curl_message=None):
info = curl.info
curl.info = None
self._multi.remove_handle(curl)
self._free_list.append(curl)
buffer = info["buffer"]
if curl_error:
error = CurlError(curl_error, curl_message)
code = error.code
body = None
effective_url = None
buffer.close()
buffer = None
else:
error = None
code = curl.getinfo(pycurl.HTTP_CODE)
effective_url = curl.getinfo(pycurl.EFFECTIVE_URL)
buffer.seek(0)
try:
info["callback"](HTTPResponse(
request=info["request"], code=code, headers=info["headers"],
buffer=buffer, effective_url=effective_url, error=error,
request_time=time.time() - info["start_time"]))
except (KeyboardInterrupt, SystemExit):
raise
except:
logging.error("Exception in callback %r", info["callback"],
exc_info=True)
示例9: response_from_conn_object
# 需要導入模塊: import pycurl [as 別名]
# 或者: from pycurl import EFFECTIVE_URL [as 別名]
def response_from_conn_object(self, conn, header, body):
# followlocation
if conn.getinfo(pycurl.EFFECTIVE_URL) != self.completeUrl:
self.setFinalUrl(conn.getinfo(pycurl.EFFECTIVE_URL))
self.totaltime = conn.getinfo(pycurl.TOTAL_TIME)
self.response = Response()
self.response.parseResponse(header, rawbody=body)
return self.response
示例10: info
# 需要導入模塊: import pycurl [as 別名]
# 或者: from pycurl import EFFECTIVE_URL [as 別名]
def info(self):
"Return a dictionary with all info on the last response."
m = {}
m['effective-url'] = self.handle.getinfo(pycurl.EFFECTIVE_URL)
m['http-code'] = self.handle.getinfo(pycurl.HTTP_CODE)
m['total-time'] = self.handle.getinfo(pycurl.TOTAL_TIME)
m['namelookup-time'] = self.handle.getinfo(pycurl.NAMELOOKUP_TIME)
m['connect-time'] = self.handle.getinfo(pycurl.CONNECT_TIME)
m['pretransfer-time'] = self.handle.getinfo(pycurl.PRETRANSFER_TIME)
m['redirect-time'] = self.handle.getinfo(pycurl.REDIRECT_TIME)
m['redirect-count'] = self.handle.getinfo(pycurl.REDIRECT_COUNT)
m['size-upload'] = self.handle.getinfo(pycurl.SIZE_UPLOAD)
m['size-download'] = self.handle.getinfo(pycurl.SIZE_DOWNLOAD)
m['speed-upload'] = self.handle.getinfo(pycurl.SPEED_UPLOAD)
m['header-size'] = self.handle.getinfo(pycurl.HEADER_SIZE)
m['request-size'] = self.handle.getinfo(pycurl.REQUEST_SIZE)
m['content-length-download'] = self.handle.getinfo(pycurl.CONTENT_LENGTH_DOWNLOAD)
m['content-length-upload'] = self.handle.getinfo(pycurl.CONTENT_LENGTH_UPLOAD)
m['content-type'] = self.handle.getinfo(pycurl.CONTENT_TYPE)
m['response-code'] = self.handle.getinfo(pycurl.RESPONSE_CODE)
m['speed-download'] = self.handle.getinfo(pycurl.SPEED_DOWNLOAD)
m['ssl-verifyresult'] = self.handle.getinfo(pycurl.SSL_VERIFYRESULT)
m['filetime'] = self.handle.getinfo(pycurl.INFO_FILETIME)
m['starttransfer-time'] = self.handle.getinfo(pycurl.STARTTRANSFER_TIME)
m['redirect-time'] = self.handle.getinfo(pycurl.REDIRECT_TIME)
m['redirect-count'] = self.handle.getinfo(pycurl.REDIRECT_COUNT)
m['http-connectcode'] = self.handle.getinfo(pycurl.HTTP_CONNECTCODE)
m['httpauth-avail'] = self.handle.getinfo(pycurl.HTTPAUTH_AVAIL)
m['proxyauth-avail'] = self.handle.getinfo(pycurl.PROXYAUTH_AVAIL)
m['os-errno'] = self.handle.getinfo(pycurl.OS_ERRNO)
m['num-connects'] = self.handle.getinfo(pycurl.NUM_CONNECTS)
m['ssl-engines'] = self.handle.getinfo(pycurl.SSL_ENGINES)
m['cookielist'] = self.handle.getinfo(pycurl.INFO_COOKIELIST)
m['lastsocket'] = self.handle.getinfo(pycurl.LASTSOCKET)
m['ftp-entry-path'] = self.handle.getinfo(pycurl.FTP_ENTRY_PATH)
return m
示例11: _update_redirected_link
# 需要導入模塊: import pycurl [as 別名]
# 或者: from pycurl import EFFECTIVE_URL [as 別名]
def _update_redirected_link(self):
try:
self._init_base_headers_curl()
self._follow_location_and_auto_referer()
self._perform_curl()
self._link = self._curl.getinfo(pycurl.EFFECTIVE_URL)
self._release_self_curl()
return True
except Exception as e:
self._make_message_and_send(str(e), True)
return False
示例12: get_final_location
# 需要導入模塊: import pycurl [as 別名]
# 或者: from pycurl import EFFECTIVE_URL [as 別名]
def get_final_location(self):
try:
self._init_base_headers_curl()
self._follow_location_and_auto_referer()
self._perform_curl()
return self._curl.getinfo(pycurl.EFFECTIVE_URL)
except Exception as e:
self._make_message_and_send(str(e), True)
return None
示例13: perform
# 需要導入模塊: import pycurl [as 別名]
# 或者: from pycurl import EFFECTIVE_URL [as 別名]
def perform(cls):
if cls._futures:
while True:
status, num_active = cls._multi.perform()
if status != pycurl.E_CALL_MULTI_PERFORM:
break
while True:
num_ready, success, fail = cls._multi.info_read()
for c in success:
cc = cls._futures.pop(c)
result = curl_result(c)
result['url'] = c._raw_url
result['id'] = c._raw_id
result['state'] = 'normal'
result['spider'] = 'pycurl'
result['payload'] = payload = c._raw_payload
# post_func = payload.get('post_func')
# if type(post_func) == str:
# post_func = load(post_func)
# if post_func:
# result = post_func(payload, result)
cc.set_result(result)
for c, err_num, err_msg in fail:
print('error:', err_num, err_msg, c.getinfo(pycurl.EFFECTIVE_URL))
result = curl_result(c)
result['url'] = c._raw_url
result['id'] = c._raw_id
result['state'] = 'error'
result['spider'] = 'pycurl'
result['error_code'] = err_num
result['error_desc'] = err_msg
result['payload'] = payload = c._raw_payload
# post_func = payload.get('post_func')
# if type(post_func) == str:
# post_func = load(post_func)
# if post_func:
# result2 = post_func(payload, result)
# if type(result2) is dict and len(result2) >= len(result):
# result = result2
cls._futures.pop(c).set_exception(CurlLoop.CurlException(code=err_num, desc=err_msg, data=result))
if num_ready == 0:
break
示例14: curl_result
# 需要導入模塊: import pycurl [as 別名]
# 或者: from pycurl import EFFECTIVE_URL [as 別名]
def curl_result(c):
effective_url = c.getinfo(pycurl.EFFECTIVE_URL)
primary_ip = c.getinfo(pycurl.PRIMARY_IP)
primary_port = c.getinfo(pycurl.PRIMARY_PORT)
local_ip = c.getinfo(pycurl.LOCAL_IP)
local_port = c.getinfo(pycurl.LOCAL_PORT)
speed_download = c.getinfo(pycurl.SPEED_DOWNLOAD)
size_download = c.getinfo(pycurl.SIZE_DOWNLOAD)
redirect_time = c.getinfo(pycurl.REDIRECT_TIME)
redirect_count = c.getinfo(pycurl.REDIRECT_COUNT)
redirect_url = c.getinfo(pycurl.REDIRECT_URL)
http_code = c.getinfo(pycurl.HTTP_CODE)
response_code = c.getinfo(pycurl.RESPONSE_CODE)
total_time = c.getinfo(pycurl.TOTAL_TIME)
content_type = c.getinfo(pycurl.CONTENT_TYPE)
namelookup_time = c.getinfo(pycurl.NAMELOOKUP_TIME)
info_filetime = c.getinfo(pycurl.INFO_FILETIME)
http_connectcode = c.getinfo(pycurl.HTTP_CONNECTCODE)
starttransfer_time = c.getinfo(pycurl.STARTTRANSFER_TIME)
pretransfer_time = c.getinfo(pycurl.PRETRANSFER_TIME)
header_size = c.getinfo(pycurl.HEADER_SIZE)
request_size = c.getinfo(pycurl.REQUEST_SIZE)
ssl_verifyresult = c.getinfo(pycurl.SSL_VERIFYRESULT)
num_connects = c.getinfo(pycurl.NUM_CONNECTS)
return {
'effective_url': effective_url,
'primary_ip': primary_ip,
'primary_port': primary_port,
'local_ip': local_ip,
'local_port': local_port,
'speed_download': speed_download,
'size_download': size_download,
'redirect_time': redirect_time,
'redirect_count': redirect_count,
'redirect_url': redirect_url,
'http_code': http_code,
'response_code': response_code,
'total_time': total_time,
'content_type': content_type,
'namelookup_time': namelookup_time,
'info_filetime': info_filetime,
'http_connectcode': http_connectcode,
'starttransfer_time': starttransfer_time,
'pretransfer_time': pretransfer_time,
'header_size': header_size,
'request_size': request_size,
'ssl_verifyresult': ssl_verifyresult,
'num_connects': num_connects,
# 'proxy_ssl_verifyresult': proxy_ssl_verifyresult,
# 'app_connecttime': app_connecttime,
}
示例15: run
# 需要導入模塊: import pycurl [as 別名]
# 或者: from pycurl import EFFECTIVE_URL [as 別名]
def run(self):
c = pycurl.Curl()
c.setopt(pycurl.URL, self.url)
c.setopt(pycurl.FOLLOWLOCATION, True)
c.setopt(pycurl.MAXREDIRS, 4)
#c.setopt(pycurl.NOBODY, 1)
c.setopt(c.VERBOSE, True)
#c.setopt(pycurl.CONNECTTIMEOUT, 20)
if self.useragent:
c.setopt(pycurl.USERAGENT, self.useragent)
# add cookies, if available
if self.cookies:
c.setopt(pycurl.COOKIE, self.cookies)
#realurl = c.getinfo(pycurl.EFFECTIVE_URL)
realurl = self.url
print("realurl",realurl)
self.filename = realurl.split("/")[-1].strip()
c.setopt(pycurl.URL, realurl)
c.setopt(pycurl.FOLLOWLOCATION, True)
c.setopt(pycurl.NOPROGRESS, False)
c.setopt(pycurl.XFERINFOFUNCTION, self.getProgress)
c.setopt(pycurl.SSL_VERIFYPEER, False)
c.setopt(pycurl.SSL_VERIFYHOST, False)
# configure pycurl output file
if self.path == False:
self.path = os.getcwd()
filepath = os.path.join(self.path, self.filename)
if os.path.exists(filepath):## remove old file,restart download
os.system("rm -rf " + filepath)
buffer = StringIO()
c.setopt(pycurl.WRITEDATA, buffer)
self._dst_path = filepath
# add cookies, if available
if self.cookies:
c.setopt(pycurl.COOKIE, self.cookies)
self._stop = False
self.progress["stopped"] = False
# download file
try:
c.perform()
except pycurl.error, error:
errno,errstr = error
print("curl error: %s" % errstr)
self._errors.append(errstr)
self._stop = True
self.progress["stopped"] = True
self.stop()