Python ProxyType.MANUAL属性代码示例

# 需要导入模块: from selenium.webdriver.common.proxy import ProxyType [as 别名]
# 或者: from selenium.webdriver.common.proxy.ProxyType import MANUAL [as 别名]
def get_html_by_webdirver(url, proxies = ''):
    html = None
    try:

        driver = webdriver.PhantomJS()

        if proxies:
            proxy=webdriver.Proxy()
            proxy.proxy_type=ProxyType.MANUAL
            proxy.http_proxy= proxies  #'220.248.229.45:3128'
            #????????webdriver.DesiredCapabilities.PHANTOMJS?
            proxy.add_to_capabilities(webdriver.DesiredCapabilities.PHANTOMJS)
            driver.start_session(webdriver.DesiredCapabilities.PHANTOMJS)

        driver.get(url)
        html = driver.page_source
        # driver.save_screenshot('1.png')   #????
        driver.close()
    except Exception as e:
        log.error(e)
    return html and len(html) < 1024 * 1024 and html or None 

# 需要导入模块: from selenium.webdriver.common.proxy import ProxyType [as 别名]
# 或者: from selenium.webdriver.common.proxy.ProxyType import MANUAL [as 别名]
def reflashProxy(caps, driver, pIPs):
    if len(pIPs) < minPIPCount:
        # ??ip???????
        pIPs = getAvailableIPs()
    # pipObj = random.choice(pIPs)
    randomPIpIndex = random.randint(0, len(pIPs))
    pipObj = pIPs[randomPIpIndex]
    pIp = pipObj[0]
    pPort = pipObj[1]
    ua = random.choice(USER_AGENTS)
    caps["phantomjs.page.settings.userAgent"] = ua
    proxy = webdriver.Proxy()
    proxy.proxy_type = ProxyType.MANUAL
    proxy.http_proxy = pIp + ':' + str(pPort)
    # ????????webdriver.DesiredCapabilities.PHANTOMJS?
    proxy.add_to_capabilities(caps)
    driver.start_session(caps)
    return pIPs, pIp, randomPIpIndex 

# 需要导入模块: from selenium.webdriver.common.proxy import ProxyType [as 别名]
# 或者: from selenium.webdriver.common.proxy.ProxyType import MANUAL [as 别名]
def set_proxy(self, proxy):
        import warnings

        warnings.warn(
            "This method has been deprecated. Please pass in the proxy object to the Driver Object",
            DeprecationWarning)
        if proxy is None:
            raise ValueError("proxy can not be None")

        if proxy.proxy_type is ProxyType.UNSPECIFIED:
            return

        self.set_preference("network.proxy.type", proxy.proxy_type['ff_value'])

        if proxy.proxy_type is ProxyType.MANUAL:
            self.set_preference("network.proxy.no_proxies_on", proxy.no_proxy)
            self._set_manual_proxy_preference("ftp", proxy.ftp_proxy)
            self._set_manual_proxy_preference("http", proxy.http_proxy)
            self._set_manual_proxy_preference("ssl", proxy.ssl_proxy)
            self._set_manual_proxy_preference("socks", proxy.socks_proxy)
        elif proxy.proxy_type is ProxyType.PAC:
            self.set_preference("network.proxy.autoconfig_url", proxy.proxy_autoconfig_url) 

# 需要导入模块: from selenium.webdriver.common.proxy import ProxyType [as 别名]
# 或者: from selenium.webdriver.common.proxy.ProxyType import MANUAL [as 别名]
def main():
    # browser = webdriver.PhantomJS()   # Be OK in command line, but not in PyCharm.
    # browser = webdriver.PhantomJS(r"/home/lxw/Downloads/phantomjs/phantomjs-2.1.1-linux-x86_64/bin/phantomjs")
    browser = webdriver.Chrome(r"/home/lxw/Software/chromedirver_selenium/chromedriver") # OK
    browser.get("http://ipecho.net/plain")
    print('session_id: ', browser.session_id)
    print('page_source: ', browser.page_source)
    print('cookie: ', browser.get_cookies())
    print("----"*10, "\n")

    # ??DesiredCapabilities(????)??????????sessionId????????????????????????????url
    proxy = webdriver.Proxy()
    proxy.proxy_type = ProxyType.MANUAL
    # req = requests.get("http://datazhiyuan.com:60001/plain", timeout=10)
    req = requests.get("http://localhost:60001/plain", timeout=10)
    print("Get an IP proxy:", req.text)
    if req.text:
        proxy.http_proxy = req.text    # '1.9.171.51:800'
    # ????????webdriver.DesiredCapabilities.PHANTOMJS?
    proxy.add_to_capabilities(webdriver.DesiredCapabilities.PHANTOMJS)
    browser.start_session(webdriver.DesiredCapabilities.PHANTOMJS)
    browser.get("http://ipecho.net/plain")
    print('session_id: ', browser.session_id)
    print('page_source: ', browser.page_source)
    print('cookie: ', browser.get_cookies())
    print("----"*10, "\n")

    # ???????
    proxy = webdriver.Proxy()
    proxy.proxy_type = ProxyType.DIRECT
    proxy.add_to_capabilities(webdriver.DesiredCapabilities.PHANTOMJS)
    browser.start_session(webdriver.DesiredCapabilities.PHANTOMJS)
    browser.get("http://ipecho.net/plain")
    print('session_id: ', browser.session_id)
    print('page_source: ', browser.page_source)
    print('cookie: ', browser.get_cookies())
    print("----"*10, "\n") 

# 需要导入模块: from selenium.webdriver.common.proxy import ProxyType [as 别名]
# 或者: from selenium.webdriver.common.proxy.ProxyType import MANUAL [as 别名]
def process_request(self, request, spider):
        if spider.name == "gsxt":
            # print("PhantomJS is starting...")
            # driver = webdriver.PhantomJS(r"/home/lxw/Downloads/phantomjs/phantomjs-2.1.1-linux-x86_64/bin/phantomjs")   # OK
            driver = webdriver.Chrome(r"/home/lxw/Software/chromedirver_selenium/chromedriver") # OK

            """
            # Using IP Proxies:
            # ????chrome?????chrome???IP?????????????????
            # ??DesiredCapabilities(????)??????????sessionId????????????????????????????url
            proxy = webdriver.Proxy()
            proxy.proxy_type = ProxyType.MANUAL
            req = requests.get("http://datazhiyuan.com:60001/plain", timeout=10)
            print("Get an IP proxy:", req.text)

            if req.text:
                proxy.http_proxy = req.text  # "1.9.171.51:800"
            # ????????webdriver.DesiredCapabilities.PHANTOMJS?
            proxy.add_to_capabilities(webdriver.DesiredCapabilities.PHANTOMJS)
            driver.start_session(webdriver.DesiredCapabilities.PHANTOMJS)
            """

            driver.get(request.url) # ????????????, ??http://roll.news.qq.com/??
            time.sleep(2)
            js = "var q=document.documentElement.scrollTop=10000"
            driver.execute_script(js)   # ???js????????????????????
            time.sleep(3)
            body = driver.page_source
            print("??" + request.url)
            return HtmlResponse(driver.current_url, body=body, encoding='utf-8', request=request)
        else:
            return 

# 需要导入模块: from selenium.webdriver.common.proxy import ProxyType [as 别名]
# 或者: from selenium.webdriver.common.proxy.ProxyType import MANUAL [as 别名]
def get_browser(self, proxy):
        """ ???????????firefox """
        # ?????
        firefox_profile = webdriver.FirefoxProfile()
        # ????image
        #firefox_profile.set_preference('permissions.default.stylesheet', 2)
        #firefox_profile.set_preference('permissions.default.image', 2)
        #firefox_profile.set_preference('dom.ipc.plugins.enabled.libflashplayer.so', 'false')
        # ??
        if proxy.is_valid():
            myProxy = '%s:%s' % (proxy.host, proxy.port)
            ff_proxy = Proxy({
                'proxyType': ProxyType.MANUAL,
                'httpProxy': myProxy,
                'ftpProxy': myProxy,
                'sslProxy': myProxy,
                'noProxy': ''})

            browser = webdriver.Firefox(firefox_profile=firefox_profile, proxy=ff_proxy)
        else:
            browser = webdriver.Firefox(firefox_profile=firefox_profile)

        return browser 

# 需要导入模块: from selenium.webdriver.common.proxy import ProxyType [as 别名]
# 或者: from selenium.webdriver.common.proxy.ProxyType import MANUAL [as 别名]
def start_browser():
    webdriver.DesiredCapabilities.PHANTOMJS['phantomjs.page.customHeaders.{}'.format('User-Agent')] = random.choice(user_agent_list)
    browser = webdriver.PhantomJS()
    # proxy = webdriver.Proxy()
    # proxy.proxy_type = ProxyType.MANUAL
    # proxy.http_proxy = '127.0.0.1:56923'
    # proxy.add_to_capabilities( webdriver.DesiredCapabilities.PHANTOMJS)
    browser.start_session(webdriver.DesiredCapabilities.PHANTOMJS)
    browser.implicitly_wait(120)
    browser.set_page_load_timeout(120)
    return browser 

# 需要导入模块: from selenium.webdriver.common.proxy import ProxyType [as 别名]
# 或者: from selenium.webdriver.common.proxy.ProxyType import MANUAL [as 别名]
def get_driver_phantomjs():
    """
    References:
    PhantomJS:
    1. [??PHANTOMJS?USER-AGENT](http://smilejay.com/2013/12/set-user-agent-for-phantomjs/)
    2. [Selenium 2 - Setting user agent for IE and Chrome](http://stackoverflow.com/questions/6940477/selenium-2-setting-user-agent-for-ie-and-chrome)
    """
    dcap = dict(DesiredCapabilities.PHANTOMJS)

    # Setting User-Agent
    ua = random.choice(RotateUserAgentMiddleware.user_agent_list)
    if ua:
        print("Current User-Agent is:", ua)
        dcap["phantomjs.page.settings.userAgent"] = ua

    driver = webdriver.PhantomJS(executable_path=r"/home/lxw/Downloads/phantomjs/phantomjs-2.1.1-linux-x86_64/bin/phantomjs", desired_capabilities=dcap)

    """
    # Setting IP Proxies
    # ??DesiredCapabilities(????)??????????sessionId????????????????????????????url
    proxy = webdriver.Proxy()
    proxy.proxy_type = ProxyType.MANUAL
    ip_proxy = get_proxy()
    if ip_proxy:
        proxy.http_proxy = ip_proxy

    # ????????webdriver.DesiredCapabilities.PHANTOMJS?
    # proxy.add_to_capabilities(DesiredCapabilities.PHANTOMJS)
    # driver.start_session(DesiredCapabilities.PHANTOMJS)
    proxy.add_to_capabilities(dcap)
    driver.start_session(dcap)
    """

    # ??????
    driver.set_page_load_timeout(TIMEOUT)
    driver.set_script_timeout(TIMEOUT)  # ???????????

    return driver 

# 需要导入模块: from selenium.webdriver.common.proxy import ProxyType [as 别名]
# 或者: from selenium.webdriver.common.proxy.ProxyType import MANUAL [as 别名]
def get_driver_chrome(self):
        # chromedriver
        options = webdriver.ChromeOptions()
        proxy = get_proxy()
        # NOTE: ??"http"?"https"??????????http?????https
        self.proxies["http"] = proxy
        self.proxies["https"] = proxy
        if proxy:
            options.add_argument('--proxy-server=' + proxy)

        display = Display(visible=0, size=(800, 800))
        display.start()
        driver = webdriver.Chrome(executable_path=r"/home/lxw/Software/chromedriver_selenium/chromedriver", chrome_options=options)
        
        """
        # PhantomJS: Not working. why?
        driver = webdriver.PhantomJS(executable_path=r"/home/lxw/Downloads/phantomjs/phantomjs-2.1.1-linux-x86_64/bin/phantomjs")
        proxy = webdriver.Proxy()
        proxy.proxy_type = ProxyType.MANUAL
        proxy_str = get_proxy()
        if proxy_str:
            proxy.http_proxy = proxy_str
        # ????????webdriver.DesiredCapabilities.PHANTOMJS?
        proxy.add_to_capabilities(webdriver.DesiredCapabilities.PHANTOMJS)
        driver.start_session(webdriver.DesiredCapabilities.PHANTOMJS)
        """

        # ??????
        driver.set_page_load_timeout(self.TIMEOUT)
        driver.set_script_timeout(self.TIMEOUT)  # ???????????
        return driver 

# 需要导入模块: from selenium.webdriver.common.proxy import ProxyType [as 别名]
# 或者: from selenium.webdriver.common.proxy.ProxyType import MANUAL [as 别名]
def setUp(self):
        myProxy = config.get('Proxy', 'host')+':'+config.get('Proxy', 'port')

        proxy = Proxy({
            'proxyType': ProxyType.MANUAL,
            'httpProxy': myProxy,
            'ftpProxy': myProxy,
            'sslProxy': myProxy,
            'noProxy': '' # set this value as desired
            })

        self.driver = webdriver.Firefox(proxy=proxy)
        self.driver.set_window_size(920, 860)
        #self.driver = WebDriver('firefox', proxy=proxy, reuse_browser=True)