Python request.install_opener方法代码示例

# 需要导入模块: from urllib import request [as 别名]
# 或者: from urllib.request import install_opener [as 别名]
def get_response(url, faker = False):
    logging.debug('get_response: %s' % url)

    # install cookies
    if cookies:
        opener = request.build_opener(request.HTTPCookieProcessor(cookies))
        request.install_opener(opener)

    if faker:
        response = request.urlopen(request.Request(url, headers = fake_headers), None)
    else:
        response = request.urlopen(url)

    data = response.read()
    if response.info().get('Content-Encoding') == 'gzip':
        data = ungzip(data)
    elif response.info().get('Content-Encoding') == 'deflate':
        data = undeflate(data)
    response.data = data
    return response

# DEPRECATED in favor of get_content() 

# 需要导入模块: from urllib import request [as 别名]
# 或者: from urllib.request import install_opener [as 别名]
def check_php_multipartform_dos(url, post_body, headers, ip):
    try:
        proxy_handler = urllib2.ProxyHandler({"http": ip})
        null_proxy_handler = urllib2.ProxyHandler({})
        opener = urllib2.build_opener(proxy_handler)
        urllib2.install_opener(opener)
        req = urllib2.Request(url)
        for key in headers.keys():
            req.add_header(key, headers[key])
        starttime = datetime.datetime.now()
        fd = urllib2.urlopen(req, post_body)
        html = fd.read()
        endtime = datetime.datetime.now()
        usetime = (endtime - starttime).seconds
        if(usetime > 5):
            result = url+" is vulnerable"
        else:
            if(usetime > 3):
                result = "need to check normal respond time"
        return [result, usetime]
    except KeyboardInterrupt:
        exit()
# end 

# 需要导入模块: from urllib import request [as 别名]
# 或者: from urllib.request import install_opener [as 别名]
def _urlretrieve(url, fpath):
    """Retrieve data from given url

    Parameters
    ----------
    url: str
        The url to the data.

    fpath: str
        The path to file where data is stored.

    """
    opener = request.build_opener()
    opener.addheaders = [("User-agent", "Mozilla/5.0")]

    with tqdm(unit="B", unit_scale=True) as progress:

        def report(chunk, chunksize, total):
            progress.total = total
            progress.update(chunksize)

        request.install_opener(opener)
        request.urlretrieve(url, fpath, reporthook=report) 

# 需要导入模块: from urllib import request [as 别名]
# 或者: from urllib.request import install_opener [as 别名]
def _checkout(self, local_dir):
        url = self.config.get("url")
        logger.info("Downloading...")
        user_agent = self.config.get("user-agent")
        if not self.config.get("verify_cert", True):
            import ssl

            ssl._create_default_https_context = ssl._create_unverified_context

        if user_agent and sys.version_info[0] >= 3:
            opener = urllib.build_opener()
            opener.addheaders = [("User-agent", user_agent)]
            urllib.install_opener(opener)
        try:
            (filename, headers) = urllib.urlretrieve(url)
        except (URLError, HTTPError) as e:
            raise RuntimeError("Failed to download '{}'. '{}'".format(url, e.reason))

        filetype = self.config.get("filetype")
        if filetype == "tar":
            t = tarfile.open(filename)
            t.extractall(local_dir)
        elif filetype == "zip":
            with zipfile.ZipFile(filename, "r") as z:
                z.extractall(local_dir)
        elif filetype == "simple":
            _filename = url.rsplit("/", 1)[1]
            os.makedirs(local_dir)
            shutil.copy2(filename, os.path.join(local_dir, _filename))
        else:
            raise RuntimeError(
                "Unknown file type '" + filetype + "' in [provider] section"
            ) 

# 需要导入模块: from urllib import request [as 别名]
# 或者: from urllib.request import install_opener [as 别名]
def getFile(cls, getfile, unpack=True):
        if cls.getProxy():
            proxy = req.ProxyHandler({'http': cls.getProxy(), 'https': cls.getProxy()})
            auth = req.HTTPBasicAuthHandler()
            opener = req.build_opener(proxy, auth, req.HTTPHandler)
            req.install_opener(opener)
        if cls.ignoreCerts():
            ctx = ssl.create_default_context()
            ctx.check_hostname = False
            ctx.verify_mode = ssl.CERT_NONE
            opener = req.build_opener(urllib.request.HTTPSHandler(context=ctx))
            req.install_opener(opener)

        response = req.urlopen(getfile)
        data = response
        # TODO: if data == text/plain; charset=utf-8, read and decode
        if unpack:
            if   'gzip' in response.info().get('Content-Type'):
                buf = BytesIO(response.read())
                data = gzip.GzipFile(fileobj=buf)
            elif 'bzip2' in response.info().get('Content-Type'):
                data = BytesIO(bz2.decompress(response.read()))
            elif 'zip' in response.info().get('Content-Type'):
                fzip = zipfile.ZipFile(BytesIO(response.read()), 'r')
                if len(fzip.namelist())>0:
                    data=BytesIO(fzip.read(fzip.namelist()[0]))
        return (data, response)


    # Feeds 

# 需要导入模块: from urllib import request [as 别名]
# 或者: from urllib.request import install_opener [as 别名]
def __configureProxy(self, proxy):
        #proxy = self.__checkProxyUrl(proxy)
        #if not proxy:
        #    raise MyExceptions.InvalidProxyUrlError()
        
        self.Utils.checkProxyConn(self.URL, proxy.netloc)
        self.Proxy = proxy
        proxyHandler = request.ProxyHandler({'http':proxy.scheme + '://' + proxy.netloc})
        opener = request.build_opener(proxyHandler)
        request.install_opener(opener)
        self.Logger.Print('Proxy ({}) has been configured.'.format(proxy.scheme + '://' + proxy.netloc)) 

# 需要导入模块: from urllib import request [as 别名]
# 或者: from urllib.request import install_opener [as 别名]
def __init__(self):
        self.DEBUG = False
        self.appid = 'wx782c26e4c19acffb'
        self.uuid = ''
        self.base_uri = ''
        self.redirect_uri = ''
        self.uin = ''
        self.sid = ''
        self.skey = ''
        self.pass_ticket = ''
        self.deviceId = 'e' + repr(random.random())[2:17]
        self.BaseRequest = {}
        self.synckey = ''
        self.SyncKey = []
        self.User = []
        self.MemberList = []
        self.ContactList = []
        self.GroupList = []
        self.autoReplyMode = False
        self.syncHost = ''

        self._handlers = dict((k, []) for k in self.message_types)
        self._handlers['location'] = []
        self._handlers['all'] = []

        self._filters = dict()

        opener = request.build_opener(request.HTTPCookieProcessor(CookieJar()))
        opener.addheaders = [('User-agent', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.109 Safari/537.36'),
                             ('Referer','https://wx2.qq.com/')]
        request.install_opener(opener) 

# 需要导入模块: from urllib import request [as 别名]
# 或者: from urllib.request import install_opener [as 别名]
def set_proxy(proxy):
    proxy_handler = request.ProxyHandler({
        'http': '%s:%s' % proxy,
        'https': '%s:%s' % proxy,
    })
    opener = request.build_opener(proxy_handler)
    request.install_opener(opener) 

# 需要导入模块: from urllib import request [as 别名]
# 或者: from urllib.request import install_opener [as 别名]
def unset_proxy():
    proxy_handler = request.ProxyHandler({})
    opener = request.build_opener(proxy_handler)
    request.install_opener(opener)

# DEPRECATED in favor of set_proxy() and unset_proxy() 

# 需要导入模块: from urllib import request [as 别名]
# 或者: from urllib.request import install_opener [as 别名]
def set_http_proxy(proxy):
    if proxy == None: # Use system default setting
        proxy_support = request.ProxyHandler()
    elif proxy == '': # Don't use any proxy
        proxy_support = request.ProxyHandler({})
    else: # Use proxy
        proxy_support = request.ProxyHandler({'http': '%s' % proxy, 'https': '%s' % proxy})
    opener = request.build_opener(proxy_support)
    request.install_opener(opener) 

# 需要导入模块: from urllib import request [as 别名]
# 或者: from urllib.request import install_opener [as 别名]
def brash(proxy_dict):
    #print(proxy_dict)
    global count
    global count1
    if count1 < 100:
        try:  #正常运行
            count = count + 1
            print(count, 'times')  #监视程序是否在正常运行，输出运行了多少次
            proxy_handler = request.ProxyHandler({'http': proxy_dict})
            opener = request.build_opener(proxy_handler)
            request.install_opener(opener)
            countUrl = len(url)
            for i in range(countUrl):  #遍历所有url
                req = request.Request(url[i], headers=head, method='POST')
                try:
                    #lock.acquire()
                    response = request.urlopen(req)  #访问网页
                    html = response.read().decode('utf-8')
                    print(html)
                    #lock.release()
                except urllib.error.URLError as e:
                    print(e.reason)
                    print("EEEEEE")
            #time.sleep(1)  #间隔执行

        except Exception:  #出现异常
            print('Retry')
            count1 = count1 + 1
            time.sleep(1)  #间隔执行
    else:
        print('much error') 

# 需要导入模块: from urllib import request [as 别名]
# 或者: from urllib.request import install_opener [as 别名]
def fetch_ktools_tar(self, location, url, attempts=3, timeout=15, cooldown=1):
        last_error = None
        proxy_config = urlrequest.getproxies()
        self.announce('Retrieving ktools from: {}'.format(url), INFO)
        self.announce('Proxy configuration: {}'.format(proxy_config), INFO)

        if proxy_config:
            # Handle Proxy config
            proxy_handler = urlrequest.ProxyHandler(proxy_config)
            opener = urlrequest.build_opener(proxy_handler)
            urlrequest.install_opener(opener)

        for i in range(attempts):
            try:
                if proxy_config:
                    # Proxied connection
                    req = urlrequest.urlopen(urlrequest.Request(url), timeout=timeout)
                    break
                else:
                    # Non proxied connection
                    req = urlrequest.urlopen(url, timeout=timeout)
                    break

            except URLError as e:
                self.announce('Fetch ktools tar failed: {} (attempt {})'.format(e, (i+1)), WARN)
                last_error = e
                sleep(cooldown)
        else:
            self.announce('Failed to get ktools tar after {} attempts'.format(attempts), ERROR)
            if last_error:
                raise last_error

        with open(location, 'wb') as f:
            f.write(req.read()) 

# 需要导入模块: from urllib import request [as 别名]
# 或者: from urllib.request import install_opener [as 别名]
def download(self, url):
        print(url)
        if not url:
            return
        ssl._create_default_https_context = ssl._create_unverified_context
        opener = urequest.build_opener()
        opener.addheaders = [
            ("Host", "tx.acgvideo.com"),
            (
                "User-Agent",
                "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36",
            ),
            ("Accept", "*/*"),
            ("Accept-Language", "en-US,en;q=0.5"),
            ("Accept-Encoding", "gzip, deflate, br"),
            ("Range", "bytes=0-"),  # Range 的值要为 bytes=0- 才能下载完整视频
            ("Referer", "https://www.bilibili.com/video/av14543079/"),
            ("Origin", "https://www.bilibili.com"),
            ("Connection", "keep-alive"),
        ]
        urequest.install_opener(opener)

        folder = self.dlpath + "/" + url[0].split("?")[0].split("/")[-1].split("-")[0]
        for i, j in enumerate(url):
            filename = j.split("?")[0].split("/")[-1]
            print(f"path: {folder}/{filename}\n")
            if not os.path.exists(folder):
                os.mkdir(folder)
            if os.path.exists(f"{folder}/{filename}"):
                self.updateProgress.emit(int(threading.current_thread().name), 100)
            else:
                urequest.urlretrieve(
                    j, filename=f"{folder}/{filename}", reporthook=self.report
                )
            self.updateSlice.emit(int(threading.current_thread().name), str(i + 1))
        if len(url) > 1:
            self.merge(folder)
        else:
            self.change2mp4(folder) 

# 需要导入模块: from urllib import request [as 别名]
# 或者: from urllib.request import install_opener [as 别名]
def check_proxy(q):
    """
    check proxy for and append to working proxies
    :param q:
    """
    if not q.empty():

        proxy = q.get(False)
        proxy = proxy.replace("\r", "").replace("\n", "")

        try:
            opener = rq.build_opener(
                rq.ProxyHandler({'https': 'https://' + proxy}),
                rq.HTTPHandler(),
                rq.HTTPSHandler()
            )

            opener.addheaders = [('User-agent', 'Mozilla/5.0')]
            rq.install_opener(opener)

            req = rq.Request('https://api.ipify.org/')

            if rq.urlopen(req).read().decode() == proxy.partition(':')[0]:
                proxys_working_list.update({proxy: proxy})
                if _verbose:
                    print(bcolors.OKGREEN + " --[+] ", proxy, " | PASS" + bcolors.ENDC)
            else:
                if _verbose:
                    print(" --[!] ", proxy, " | FAILED")

        except Exception as err:
            if _verbose:
                print(" --[!] ", proxy, " | FAILED")
            if _debug:
                logger.error(err)
            pass 

# 需要导入模块: from urllib import request [as 别名]
# 或者: from urllib.request import install_opener [as 别名]
def get_csrf():
    """
    get CSRF token from login page to use in POST requests
    """
    global csrf_token

    print(bcolors.WARNING + "[+] Getting CSRF Token: " + bcolors.ENDC)

    try:
        opener = rq.build_opener(rq.HTTPHandler(), rq.HTTPSHandler())
        opener.addheaders = [('User-agent', 'Mozilla/5.0')]
        rq.install_opener(opener)

        request = rq.Request('https://www.instagram.com/')
        try:
            # python 2
            headers = rq.urlopen(request).info().headers
        except Exception:
            # python 3
            headers = rq.urlopen(request).info().get_all('Set-Cookie')

        for header in headers:
            if header.find('csrftoken') != -1:
                csrf_token = header.partition(';')[0].partition('=')[2]
                print(bcolors.OKGREEN + "[+] CSRF Token :", csrf_token, "\n" + bcolors.ENDC)
    except Exception as err:
        print(bcolors.FAIL + "[!] Can't get CSRF token , please use -d for debug" + bcolors.ENDC)

        if _debug:
            logger.error(err)

        print(bcolors.FAIL + "[!] Exiting..." + bcolors.ENDC)
        exit(3)