Python request.ProxyHandler方法代码示例

# 需要导入模块: from urllib import request [as 别名]
# 或者: from urllib.request import ProxyHandler [as 别名]
def do_socks(self, line):
        headers = ["Protocol", "Target", "Username", "AdminStatus", "Port"]
        url = "http://localhost:9090/ntlmrelayx/api/v1.0/relays"
        try:
            proxy_handler = ProxyHandler({})
            opener = build_opener(proxy_handler)
            response = Request(url)
            r = opener.open(response)
            result = r.read()
            items = json.loads(result)
        except Exception as e:
            logging.error("ERROR: %s" % str(e))
        else:
            if len(items) > 0:
                self.printTable(items, header=headers)
            else:
                logging.info('No Relays Available!') 

# 需要导入模块: from urllib import request [as 别名]
# 或者: from urllib.request import ProxyHandler [as 别名]
def scrape(category_name,commodity_name):
    
    #i use proxy handler cuz my uni network runs on its proxy
    #and i cannot authenticate python through the proxy
    #so i use empty proxy to bypass the authentication
    proxy_handler = u.ProxyHandler({})
    opener = u.build_opener(proxy_handler)
    
    #cme officially forbids scraping
    #so a header must be used for disguise as an internet browser
    #the developers say no to scraping, it appears to be so
    #but actually they turn a blind eye to us, thx
    #i need different types of commodity
    #so i need to format the website for each commodity
    req=u.Request('http://www.cmegroup.com/trading/metals/%s/%s.html'%(
            category_name,commodity_name),headers={'User-Agent': 'Mozilla/5.0'})
    response=opener.open(req)
    result=response.read()
    soup=bs(result,'html.parser')
    
    return soup


# 

# 需要导入模块: from urllib import request [as 别名]
# 或者: from urllib.request import ProxyHandler [as 别名]
def GetHandlers(self):
    """Retrieve the appropriate urllib handlers for the given configuration.

    Returns:
      A list of urllib.request.BaseHandler subclasses to be used when making
      calls with proxy.
    """
    handlers = []

    if self.ssl_context:
      handlers.append(HTTPSHandler(context=self.ssl_context))

    if self.proxies:
      handlers.append(ProxyHandler(self.proxies))

    return handlers 

# 需要导入模块: from urllib import request [as 别名]
# 或者: from urllib.request import ProxyHandler [as 别名]
def check_php_multipartform_dos(url, post_body, headers, ip):
    try:
        proxy_handler = urllib2.ProxyHandler({"http": ip})
        null_proxy_handler = urllib2.ProxyHandler({})
        opener = urllib2.build_opener(proxy_handler)
        urllib2.install_opener(opener)
        req = urllib2.Request(url)
        for key in headers.keys():
            req.add_header(key, headers[key])
        starttime = datetime.datetime.now()
        fd = urllib2.urlopen(req, post_body)
        html = fd.read()
        endtime = datetime.datetime.now()
        usetime = (endtime - starttime).seconds
        if(usetime > 5):
            result = url+" is vulnerable"
        else:
            if(usetime > 3):
                result = "need to check normal respond time"
        return [result, usetime]
    except KeyboardInterrupt:
        exit()
# end 

# 需要导入模块: from urllib import request [as 别名]
# 或者: from urllib.request import ProxyHandler [as 别名]
def download(self, url, retry_count=3, headers=None, proxy=None, data=None):
        if url is None:
            return None
        try:
            req = request.Request(url, headers=headers, data=data)
            cookie = cookiejar.CookieJar()
            cookie_process = request.HTTPCookieProcessor(cookie)
            opener = request.build_opener()
            if proxy:
                proxies = {urlparse(url).scheme: proxy}
                opener.add_handler(request.ProxyHandler(proxies))
            content = opener.open(req).read()
        except error.URLError as e:
            print('HtmlDownLoader download error:', e.reason)
            content = None
            if retry_count > 0:
                if hasattr(e, 'code') and 500 <= e.code < 600:
                    #说明是 HTTPError 错误且 HTTP CODE 为 5XX 范围说明是服务器错误，可以尝试再次下载
                    return self.download(url, retry_count-1, headers, proxy, data)
        return content 

# 需要导入模块: from urllib import request [as 别名]
# 或者: from urllib.request import ProxyHandler [as 别名]
def _get_roaming_xvrttoken(self):
        """Get a X-VRT-Token for roaming"""
        vrtlogin_at = self.get_token('vrtlogin-at')
        if vrtlogin_at is None:
            return None
        cookie_value = 'vrtlogin-at=' + vrtlogin_at
        headers = {'Cookie': cookie_value}
        opener = build_opener(NoRedirection, ProxyHandler(self._proxies))
        log(2, 'URL get: {url}', url=unquote(self._ROAMING_TOKEN_GATEWAY_URL))
        req = Request(self._ROAMING_TOKEN_GATEWAY_URL, headers=headers)
        req_info = opener.open(req).info()
        cookie_value += '; state=' + req_info.get('Set-Cookie').split('state=')[1].split('; ')[0]
        url = req_info.get('Location')
        log(2, 'URL get: {url}', url=unquote(url))
        url = opener.open(url).info().get('Location')
        headers = {'Cookie': cookie_value}
        if url is None:
            return None
        req = Request(url, headers=headers)
        log(2, 'URL get: {url}', url=unquote(url))
        setcookie_header = opener.open(req).info().get('Set-Cookie')
        return TokenResolver._create_token_dictionary(setcookie_header) 

# 需要导入模块: from urllib import request [as 别名]
# 或者: from urllib.request import ProxyHandler [as 别名]
def __init__(self, protocol, drm=None):
        """Initialize InputStream Helper class"""

        self.protocol = protocol
        self.drm = drm

        from platform import uname
        log(0, 'Platform information: {uname}', uname=uname())

        if self.protocol not in config.INPUTSTREAM_PROTOCOLS:
            raise InputStreamException('UnsupportedProtocol')

        self.inputstream_addon = config.INPUTSTREAM_PROTOCOLS[self.protocol]

        if self.drm:
            if self.drm not in config.DRM_SCHEMES:
                raise InputStreamException('UnsupportedDRMScheme')

            self.drm = config.DRM_SCHEMES[drm]

        # Add proxy support to HTTP requests
        proxies = get_proxies()
        if proxies:
            try:  # Python 3
                from urllib.request import build_opener, install_opener, ProxyHandler
            except ImportError:  # Python 2
                from urllib2 import build_opener, install_opener, ProxyHandler
            install_opener(build_opener(ProxyHandler(proxies))) 

# 需要导入模块: from urllib import request [as 别名]
# 或者: from urllib.request import ProxyHandler [as 别名]
def getFile(cls, getfile, unpack=True):
        if cls.getProxy():
            proxy = req.ProxyHandler({'http': cls.getProxy(), 'https': cls.getProxy()})
            auth = req.HTTPBasicAuthHandler()
            opener = req.build_opener(proxy, auth, req.HTTPHandler)
            req.install_opener(opener)
        if cls.ignoreCerts():
            ctx = ssl.create_default_context()
            ctx.check_hostname = False
            ctx.verify_mode = ssl.CERT_NONE
            opener = req.build_opener(urllib.request.HTTPSHandler(context=ctx))
            req.install_opener(opener)

        response = req.urlopen(getfile)
        data = response
        # TODO: if data == text/plain; charset=utf-8, read and decode
        if unpack:
            if   'gzip' in response.info().get('Content-Type'):
                buf = BytesIO(response.read())
                data = gzip.GzipFile(fileobj=buf)
            elif 'bzip2' in response.info().get('Content-Type'):
                data = BytesIO(bz2.decompress(response.read()))
            elif 'zip' in response.info().get('Content-Type'):
                fzip = zipfile.ZipFile(BytesIO(response.read()), 'r')
                if len(fzip.namelist())>0:
                    data=BytesIO(fzip.read(fzip.namelist()[0]))
        return (data, response)


    # Feeds 

# 需要导入模块: from urllib import request [as 别名]
# 或者: from urllib.request import ProxyHandler [as 别名]
def __init__(self, _=None):
        """Build self._urllib_opener"""

        proxy = super().__str__()

        if proxy == "None":
            self._urllib_opener = build_opener()
            return

        components = list(re.match(self._match_regexp, proxy).groups())
        self.scheme, self.host, self.port = components
        self.components = components

        if self.scheme == "socks4":
            socks4_handler = SocksiPyHandler(socks.PROXY_TYPE_SOCKS4,
                                             self.host,
                                             int(self.port))
            self._urllib_opener = build_opener(socks4_handler)
        elif self.scheme == "socks5":
            socks5_handler = SocksiPyHandler(socks.PROXY_TYPE_SOCKS5,
                                             self.host,
                                             int(self.port))
            self._urllib_opener = build_opener(socks5_handler)
        else:
            proxy_handler = ProxyHandler({'http': proxy, 'https': proxy})
            self._urllib_opener = build_opener(proxy_handler) 

# 需要导入模块: from urllib import request [as 别名]
# 或者: from urllib.request import ProxyHandler [as 别名]
def make_http_call(callname, data):
    """Make an IPC call via HTTP and wait for it to return.
    The contents of data will be expanded to kwargs and passed into the target function."""
    from collections import OrderedDict
    try:  # Python 3
        from urllib.request import build_opener, install_opener, ProxyHandler, HTTPError, URLError, urlopen
    except ImportError:  # Python 2
        from urllib2 import build_opener, install_opener, ProxyHandler, HTTPError, URLError, urlopen
    import json
    debug('Handling HTTP IPC call to {}'.format(callname))
    # Note: On python 3, using 'localhost' slowdown the call (Windows OS is affected) not sure if it is an urllib issue
    url = 'http://127.0.0.1:{}/{}'.format(g.LOCAL_DB.get_value('ns_service_port', 8001), callname)
    install_opener(build_opener(ProxyHandler({})))  # don't use proxy for localhost
    try:
        result = json.loads(
            urlopen(url=url, data=json.dumps(data).encode('utf-8'), timeout=IPC_TIMEOUT_SECS).read(),
            object_pairs_hook=OrderedDict)
    except HTTPError as exc:
        result = json.loads(exc.reason)
    except URLError as exc:
        # On PY2 the exception message have to be decoded with latin-1 for system with symbolic characters
        err_msg = g.py2_decode(str(exc), 'latin-1')
        if '10049' in err_msg:
            err_msg += '\r\nPossible cause is wrong localhost settings in your operative system.'
        error(err_msg)
        raise BackendNotReady(g.py2_encode(err_msg, encoding='latin-1'))
    _raise_for_error(callname, result)
    return result 

# 需要导入模块: from urllib import request [as 别名]
# 或者: from urllib.request import ProxyHandler [as 别名]
def make_http_call_cache(callname, params, data):
    """Make an IPC call via HTTP and wait for it to return.
    The contents of data will be expanded to kwargs and passed into the target function."""
    try:  # Python 3
        from urllib.request import build_opener, install_opener, ProxyHandler, HTTPError, URLError, Request, urlopen
    except ImportError:  # Python 2
        from urllib2 import build_opener, install_opener, ProxyHandler, HTTPError, URLError, Request, urlopen
    import json
    # debug('Handling HTTP IPC call to {}'.format(callname))
    # Note: On python 3, using 'localhost' slowdown the call (Windows OS is affected) not sure if it is an urllib issue
    url = 'http://127.0.0.1:{}/{}'.format(g.LOCAL_DB.get_value('cache_service_port', 8002), callname)
    install_opener(build_opener(ProxyHandler({})))  # don't use proxy for localhost
    r = Request(url=url, data=data, headers={'Params': json.dumps(params)})
    try:
        result = urlopen(r, timeout=IPC_TIMEOUT_SECS).read()
    except HTTPError as exc:
        try:
            raise apierrors.__dict__[exc.reason]()
        except KeyError:
            raise Exception('The service has returned: {}'.format(exc.reason))
    except URLError as exc:
        # On PY2 the exception message have to be decoded with latin-1 for system with symbolic characters
        err_msg = g.py2_decode(str(exc), 'latin-1')
        if '10049' in err_msg:
            err_msg += '\r\nPossible cause is wrong localhost settings in your operative system.'
        error(err_msg)
        raise BackendNotReady(g.py2_encode(err_msg, encoding='latin-1'))
    return result 

# 需要导入模块: from urllib import request [as 别名]
# 或者: from urllib.request import ProxyHandler [as 别名]
def __configureProxy(self, proxy):
        #proxy = self.__checkProxyUrl(proxy)
        #if not proxy:
        #    raise MyExceptions.InvalidProxyUrlError()
        
        self.Utils.checkProxyConn(self.URL, proxy.netloc)
        self.Proxy = proxy
        proxyHandler = request.ProxyHandler({'http':proxy.scheme + '://' + proxy.netloc})
        opener = request.build_opener(proxyHandler)
        request.install_opener(opener)
        self.Logger.Print('Proxy ({}) has been configured.'.format(proxy.scheme + '://' + proxy.netloc)) 

# 需要导入模块: from urllib import request [as 别名]
# 或者: from urllib.request import ProxyHandler [as 别名]
def set_proxy(proxy):
    proxy_handler = request.ProxyHandler({
        'http': '%s:%s' % proxy,
        'https': '%s:%s' % proxy,
    })
    opener = request.build_opener(proxy_handler)
    request.install_opener(opener) 

# 需要导入模块: from urllib import request [as 别名]
# 或者: from urllib.request import ProxyHandler [as 别名]
def unset_proxy():
    proxy_handler = request.ProxyHandler({})
    opener = request.build_opener(proxy_handler)
    request.install_opener(opener)

# DEPRECATED in favor of set_proxy() and unset_proxy() 

# 需要导入模块: from urllib import request [as 别名]
# 或者: from urllib.request import ProxyHandler [as 别名]
def set_http_proxy(proxy):
    if proxy == None: # Use system default setting
        proxy_support = request.ProxyHandler()
    elif proxy == '': # Don't use any proxy
        proxy_support = request.ProxyHandler({})
    else: # Use proxy
        proxy_support = request.ProxyHandler({'http': '%s' % proxy, 'https': '%s' % proxy})
    opener = request.build_opener(proxy_support)
    request.install_opener(opener)