Python request.HTTPCookieProcessor方法代码示例

# 需要导入模块: from urllib import request [as 别名]
# 或者: from urllib.request import HTTPCookieProcessor [as 别名]
def __init__(
        self,
        host,
        port=8069,
        timeout=120,
        version=None,
        deserialize=True,
        opener=None,
    ):
        super(ConnectorJSONRPC, self).__init__(host, port, timeout, version)
        self.deserialize = deserialize
        # One URL opener (with cookies handling) shared between
        # JSON and HTTP requests
        if opener is None:
            cookie_jar = CookieJar()
            opener = build_opener(HTTPCookieProcessor(cookie_jar))
        self._opener = opener
        self._proxy_json, self._proxy_http = self._get_proxies() 

# 需要导入模块: from urllib import request [as 别名]
# 或者: from urllib.request import HTTPCookieProcessor [as 别名]
def __init__(self):
        self.articles = []
        self.query = None
        self.cjar = MozillaCookieJar()

        # If we have a cookie file, load it:
        if ScholarConf.COOKIE_JAR_FILE and \
           os.path.exists(ScholarConf.COOKIE_JAR_FILE):
            try:
                self.cjar.load(ScholarConf.COOKIE_JAR_FILE,
                               ignore_discard=True)
                ScholarUtils.log('info', 'loaded cookies file')
            except Exception as msg:
                ScholarUtils.log('warn', 'could not load cookies file: %s' % msg)
                self.cjar = MozillaCookieJar() # Just to be safe

        self.opener = build_opener(HTTPCookieProcessor(self.cjar))
        self.settings = None # Last settings object, if any 

# 需要导入模块: from urllib import request [as 别名]
# 或者: from urllib.request import HTTPCookieProcessor [as 别名]
def download(self, url, retry_count=3, headers=None, proxy=None, data=None):
        if url is None:
            return None
        try:
            req = request.Request(url, headers=headers, data=data)
            cookie = cookiejar.CookieJar()
            cookie_process = request.HTTPCookieProcessor(cookie)
            opener = request.build_opener()
            if proxy:
                proxies = {urlparse(url).scheme: proxy}
                opener.add_handler(request.ProxyHandler(proxies))
            content = opener.open(req).read()
        except error.URLError as e:
            print('HtmlDownLoader download error:', e.reason)
            content = None
            if retry_count > 0:
                if hasattr(e, 'code') and 500 <= e.code < 600:
                    #说明是 HTTPError 错误且 HTTP CODE 为 5XX 范围说明是服务器错误，可以尝试再次下载
                    return self.download(url, retry_count-1, headers, proxy, data)
        return content 

# 需要导入模块: from urllib import request [as 别名]
# 或者: from urllib.request import HTTPCookieProcessor [as 别名]
def __request__(self):

        Cookiejar = CookieJar()
        opener = build_opener(HTTPCookieProcessor(Cookiejar))
        _header = dict(self.headers.items())
        if self.cookie:
            _header.update({'Cookie': self.cookie})
        req = Request(self.url, headers=_header, origin_req_host=self.host)

        res = None

        try:
            res = opener.open(req)
            # break
        except HTTPError as e:
            # if e.code >= 400 and e.code < 500:
            return e, None

        except (socket.timeout, URLError) as e:
            return e, None
        except Exception as e:
            traceback.print_exc()
            return e, None

        return res, Cookiejar._cookies 

# 需要导入模块: from urllib import request [as 别名]
# 或者: from urllib.request import HTTPCookieProcessor [as 别名]
def cookie_friendly_download(referer_url, file_url, store_dir='.', timeout=1000):
    from http.cookiejar import CookieJar
    from urllib import request
    cj = CookieJar()
    cp = request.HTTPCookieProcessor(cj)
    opener = request.build_opener(cp)
    with opener.open(referer_url) as fin:
        fin.headers.items()
    import os
    from os import path
    with opener.open(file_url, timeout=timeout) as fin:
        file_bin = fin.read()
        filename = fin.headers['Content-Disposition']
        filename = filename.split(';')[-1].split('=')[1]
        os.makedirs(store_dir, exist_ok=True)
        with open(path.join(store_dir, filename), mode='wb') as fout:
            fout.write(file_bin)
            return path.join(store_dir, filename) 

# 需要导入模块: from urllib import request [as 别名]
# 或者: from urllib.request import HTTPCookieProcessor [as 别名]
def __init__(self, host, port, timeout=120, ssl=False, opener=None):
        self._root_url = "{http}{host}:{port}".format(
            http=(ssl and "https://" or "http://"), host=host, port=port
        )
        self._timeout = timeout
        self._builder = URLBuilder(self)
        self._opener = opener
        if not opener:
            cookie_jar = CookieJar()
            self._opener = build_opener(HTTPCookieProcessor(cookie_jar)) 

# 需要导入模块: from urllib import request [as 别名]
# 或者: from urllib.request import HTTPCookieProcessor [as 别名]
def __init__(self, base_url="", tag="", cookiejar=None, debug=False):
        self.tag = tag
        hh  = urllib2.HTTPHandler( debuglevel=1 if debug else 0)
        hsh = urllib2.HTTPSHandler(debuglevel=1 if debug else 0)
        cp  = urllib2.HTTPCookieProcessor(cookiejar)
        self._opener = urllib2.build_opener(hh, hsh, cp)
        scheme, netloc, path, q, f  = urlparse.urlsplit(base_url, "http")
        if not netloc:
            netloc, _, path = path.partition('/')
        self.base_url = urlparse.urlunsplit((scheme, netloc, path, q, f)) 

# 需要导入模块: from urllib import request [as 别名]
# 或者: from urllib.request import HTTPCookieProcessor [as 别名]
def nse_opener(self):
        """
        builds opener for urllib2
        :return: opener object
        """
        cj = CookieJar()
        return build_opener(HTTPCookieProcessor(cj)) 

# 需要导入模块: from urllib import request [as 别名]
# 或者: from urllib.request import HTTPCookieProcessor [as 别名]
def __init__(self, host, port, timeout=120, ssl=False, opener=None):
        self._root_url = "{http}{host}:{port}".format(
            http=(ssl and "https://" or "http://"), host=host, port=port)
        self._timeout = timeout
        self._builder = URLBuilder(self)
        self._opener = opener
        if not opener:
            cookie_jar = CookieJar()
            self._opener = build_opener(HTTPCookieProcessor(cookie_jar)) 

# 需要导入模块: from urllib import request [as 别名]
# 或者: from urllib.request import HTTPCookieProcessor [as 别名]
def __init__(self):
        self.cookiejar = CookieJar()
        self._cookie_processor = HTTPCookieProcessor(self.cookiejar)
        self.form = None

        self.url = "http://0.0.0.0:8080/"
        self.path = "/"
        
        self.status = None
        self.data = None
        self._response = None
        self._forms = None 

# 需要导入模块: from urllib import request [as 别名]
# 或者: from urllib.request import HTTPCookieProcessor [as 别名]
def setOpener(self):
        """ 初始化opener
        """
        cj = cookiejar.CookieJar()
        pro = request.HTTPCookieProcessor(cj)
        self.opener = request.build_opener(pro)
        header = []
        for key, value in self._headers.items():
            elem = (key, value)
            header.append(elem)
        self.opener.addheaders = header 

# 需要导入模块: from urllib import request [as 别名]
# 或者: from urllib.request import HTTPCookieProcessor [as 别名]
def create_cookie_opener(self):
        '''
        设置启用Cookie
        :return: 返回一个自定义的opener
        '''
        cookie = cookiejar.CookieJar()
        cookie_process = request.HTTPCookieProcessor(cookie)
        opener = request.build_opener(cookie_process)
        return opener 

# 需要导入模块: from urllib import request [as 别名]
# 或者: from urllib.request import HTTPCookieProcessor [as 别名]
def __init__(self):
        super(WebService, self).__init__()
        self.cache = defaultdict(defaultdict)
        self._cookie = CookieJar()
        self._opener = urllib2.build_opener(
            urllib2.HTTPCookieProcessor(self._cookie))
        self.query_interval = 1 

# 需要导入模块: from urllib import request [as 别名]
# 或者: from urllib.request import HTTPCookieProcessor [as 别名]
def __init__(self):
        super(WebService, self).__init__()
        self._cookie = CookieJar()
        self._opener = urllib2.build_opener(
            urllib2.HTTPCookieProcessor(self._cookie))
        self.query_interval = 1.0 

# 需要导入模块: from urllib import request [as 别名]
# 或者: from urllib.request import HTTPCookieProcessor [as 别名]
def initOpener(self):
        self.cookiejar = CookieJar()
        self.opener = build_opener(HTTPCookieProcessor(self.cookiejar))
        self.opener.addheaders = list(self.headers.items())