Python webdriver.PhantomJS方法代码示例

# 需要导入模块: from selenium import webdriver [as 别名]
# 或者: from selenium.webdriver import PhantomJS [as 别名]
def loadmap(self):
        '''
        loadmap()
        Creates a browser object and loads the webpage.
        It sets up the map to the proper zoom level.

        Returns the browser on success, None on fail.
        '''
        browser = webdriver.PhantomJS(desired_capabilities={'phantomjs.page.settings.resourceTimeout': '20000'})
        browser.set_window_size(abovetustin_image_width, abovetustin_image_height)

        print("getting web page {}".format(self.url))
        browser.set_page_load_timeout(15)
        browser.get(self.url)

        # Need to wait for the page to load
        timeout = g_request_timeout
        print ("waiting for page to load...")
        wait = WebDriverWait(browser, timeout)
        element = wait.until(EC.element_to_be_clickable((By.CLASS_NAME,'vrsMenu')))
        self.browser = browser 

# 需要导入模块: from selenium import webdriver [as 别名]
# 或者: from selenium.webdriver import PhantomJS [as 别名]
def my_run(self, page):
        raw_url = "http://www.kuaidaili.com/proxylist/{page}/"
        url = raw_url.replace("{page}", str(page))
        logger.debug(url)
        driver = webdriver.PhantomJS(executable_path=self.phantomjs_path)
        driver.get(url)
        raw_html = driver.execute_script("return document.getElementsByTagName('html')[0].innerHTML")

        soup = BeautifulSoup(raw_html, "html5lib")
        t_result = list()
        for tr in soup.find_all("tr")[1:]:
            each_item = {}
            td = tr.find_all("td")

            # 填充数据
            each_item['ip'] = td[0].get_text()
            each_item['port'] = td[1].get_text()
            each_item['type'] = td[2].get_text()
            each_item['protocol'] = td[3].get_text().replace(", ", "-")
            each_item['location'] = td[5].get_text()
            each_item['time'] = filter(lambda ch: ch in '0123456789.', td[6].get_text().encode("utf8"))
            t_result.append(each_item)
        return t_result 

# 需要导入模块: from selenium import webdriver [as 别名]
# 或者: from selenium.webdriver import PhantomJS [as 别名]
def __init__(self):
        self.browse = None
        self.source_playlist = None
        self.target_playlist_tag = None
        self.success_list = list()
        self.failed_list = list()
        os.environ["webdriver.chrome.driver"] = chrome_driver_path
        os.environ["webdriver.phantomjs.driver"] = phantomjs_driver_path
        # chromedriver = chrome_driver_path
        phantomjs_driver = phantomjs_driver_path

        opts = Options()
        opts.add_argument("user-agent={}".format(headers["User-Agent"]))
        # browser = webdriver.Chrome(chromedriver)
        browser = webdriver.PhantomJS(phantomjs_driver)
        self.browser = browser
        self.wait = ui.WebDriverWait(self.browser, 5)
        self.config = Config() 

# 需要导入模块: from selenium import webdriver [as 别名]
# 或者: from selenium.webdriver import PhantomJS [as 别名]
def fulltext_extract(bookmark):
    browser = webdriver.PhantomJS(service_args=[
        "--ignore-ssl-errors=true",
        "--ssl-protocol=tlsv1",
        "--load-images=no"])
    fulltext_bookmark = Bookmark.query.get(bookmark.id)
    browser.get(fulltext_bookmark.main_url)
    body = browser.find_element_by_tag_name('body')
    bodytext = body.text
    soup = BeautifulSoup4(bodytext)
    full_text = soup.text
    full_text = " ".join(full_text.split())
    full_text = full_text.replace('\n', '')
    full_text = full_text.encode('utf-8')
    fulltext_bookmark.full_text = full_text
    db.session.commit()
    browser.quit() 

# 需要导入模块: from selenium import webdriver [as 别名]
# 或者: from selenium.webdriver import PhantomJS [as 别名]
def selenium_recruits(request, recruitment_loop):
    def recruits():
        for url in recruitment_loop:
            kwargs = {}
            driver_class = DRIVER_MAP.get(request.param, webdriver.PhantomJS)
            if driver_class is webdriver.PhantomJS:
                # PhantomJS needs a new local storage for every run
                tmpdirname = tempfile.mkdtemp()
                kwargs = {
                    "service_args": ["--local-storage-path={}".format(tmpdirname)],
                }
            driver = driver_class(**kwargs)
            driver.get(url)
            try:
                yield driver
            finally:
                try:
                    driver.quit()
                except Exception:
                    pass

    yield recruits() 

# 需要导入模块: from selenium import webdriver [as 别名]
# 或者: from selenium.webdriver import PhantomJS [as 别名]
def getWebdriver(self, driverType):
		if driverType == 'proxy':
			profile = webdriver.FirefoxProfile()
			profile.set_preference( "network.proxy.type", 1 )
			profile.set_preference( "network.proxy.socks", "127.0.0.1" )
			profile.set_preference( "network.proxy.socks_port", 9150 )
			profile.set_preference( "network.proxy.socks_remote_dns", True )
			profile.set_preference( "places.history.enabled", False )
			profile.set_preference( "privacy.clearOnShutdown.offlineApps", True )
			profile.set_preference( "privacy.clearOnShutdown.passwords", True )
			profile.set_preference( "privacy.clearOnShutdown.siteSettings", True )
			profile.set_preference( "privacy.sanitize.sanitizeOnShutdown", True )
			profile.set_preference( "signon.rememberSignons", False )
			profile.set_preference( "network.cookie.lifetimePolicy", 2 )
			profile.set_preference( "network.dns.disablePrefetch", True )
			profile.set_preference( "network.http.sendRefererHeader", 0 )
			profile.set_preference( "javascript.enabled", False )
			profile.set_preference( "permissions.default.image", 2 )
			return webdriver.Firefox(profile)
		elif driverType == 'headless':
			return webdriver.PhantomJS()
		else:
			return webdriver.Firefox() 

# 需要导入模块: from selenium import webdriver [as 别名]
# 或者: from selenium.webdriver import PhantomJS [as 别名]
def get_driver(self):
        # Start a new browser and return the WebDriver

        browser_name = self.config.get('selenium', 'browser')

        if browser_name == 'firefox':
            from selenium.webdriver import Firefox

            return Firefox()

        if browser_name == 'chrome':
            from selenium.webdriver import Chrome

            return Chrome()

        if browser_name == 'phantomjs':
            from selenium.webdriver import PhantomJS

            return PhantomJS()

        # @TODO: Add chrome
        raise RuntimeError('Unsupported/unknown browser') 

# 需要导入模块: from selenium import webdriver [as 别名]
# 或者: from selenium.webdriver import PhantomJS [as 别名]
def Single_Issue(url,Quality):
	#print url
	print 'Quality To Download : ',Quality[0]
	print 'Order To Download : ',Quality[1]
	#sys.exit()
	#print url,' This is first'
	
	browser = webdriver.PhantomJS(service_args=['--load-images=no'])
	browser.get(url)
	try:
		element = WebDriverWait(browser, 10).until(
			EC.presence_of_element_located((By.ID, "stSegmentFrame"))
		)
		#print 'Downloading the whole page! Will take some time, please don\'t close this script...\n'
		#print 'I\'ve waited long enough'
	except Exception, e:
		#raise e
		browser.save_screenshot('Single_exception.png')
		print e
		pass 

# 需要导入模块: from selenium import webdriver [as 别名]
# 或者: from selenium.webdriver import PhantomJS [as 别名]
def createHeadlessBrowser(proxy=None, XResolution=1024, YResolution=768, timeout=20):
	#proxy = None
	if TEST_MODE == False:
		dcap = dict(DesiredCapabilities.PHANTOMJS)
		dcap["phantomjs.page.settings.userAgent"] = (
		    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.86 Safari/537.36')
		# Fake browser headers
		if proxy != None:
			# This means the user set a proxy
			service_args = ['--proxy={}'.format(proxy),'--proxy-type=https','--ignore-ssl-errors=true', '--ssl-protocol=any', '--web-security=false',]
			driver = webdriver.PhantomJS(service_args=service_args, desired_capabilities=dcap)
		else:
			# No proxy was set by the user
			driver = webdriver.PhantomJS(desired_capabilities=dcap)
		driver.set_window_size(XResolution,YResolution)
		# Sets the screen resolution
		# Ideally this will be dynamic based on the number of browsers open
		driver.set_page_load_timeout(timeout)
		# Sets the timeout for the selenium window
	else:
		driver = webdriver.Firefox()
	return driver
	# Returns driver instance 

# 需要导入模块: from selenium import webdriver [as 别名]
# 或者: from selenium.webdriver import PhantomJS [as 别名]
def get_latest_wallpapers():
    browser = webdriver.PhantomJS(PHANTOMJS_PATH, service_args=['--ignore-ssl-errors=true', '--ssl-protocol=any'])
    today_date = time.strftime("%d+%b+%Y")  
    yesterday = datetime.now() - timedelta(days=1)
    yesterday_date = yesterday.strftime('%d+%b+%Y')
    first_page_url = 'http://www.espncricinfo.com/ci/content/image/?datefrom='+yesterday_date+'&dateupto='+today_date+';'
    browser.get(first_page_url)
    wait = WebDriverWait(browser, 10)
    wait.until(EC.visibility_of_element_located((By.CLASS_NAME, "img-wrap")))
    time.sleep(2)
    # let's parse our html
    soup = BeautifulSoup(browser.page_source, "html.parser")
    images = soup.find_all('div', class_='picture')
    for image in images:
        url = "http://www.espncricinfo.com/" + image.find('a').get('href')
        print(url) 

# 需要导入模块: from selenium import webdriver [as 别名]
# 或者: from selenium.webdriver import PhantomJS [as 别名]
def driver(self, browser):
        if browser == 'firefox':
            return webdriver.Firefox()
        if browser == 'chrome':
            return webdriver.Chrome('chromedriver.exe')
        if browser == 'opera':
            # TODO: Opera implementation is quite buggy annoyingly. It won't close at the moment
            # Need to investigate.
            options = webdriver.ChromeOptions()
            options.binary_location = "C:\\Program Files\\Opera\\launcher.exe"
            return  webdriver.Opera(executable_path='operadriver.exe', opera_options=options)
        if browser == 'ie':
            return webdriver.Ie()
        if browser == 'edge':
            # TODO: check for Windows < 8.1?
            return webdriver.Edge()
        if browser == 'phantom':
            return webdriver.PhantomJS()
        raise XVEx("{} is not supported on {}".format(browser, self._device.os_name())) 

# 需要导入模块: from selenium import webdriver [as 别名]
# 或者: from selenium.webdriver import PhantomJS [as 别名]
def __init__(self, *args, **kwargs):
        super(FormSpider, self).__init__(*args, **kwargs)

        self.start_urls = [kwargs.get('start_url')]

        follow = True if kwargs.get('follow') == 'true' else False
        self.rules = (
            Rule (SgmlLinkExtractor(allow=('')), callback='parse_form', follow=follow),
        )
        super(FormSpider, self)._compile_rules()

        try:
            proxy = kwargs.get('proxy')
            service_args = [
                '--proxy=' + proxy,
                '--proxy-type=http',
            ]
        except:
            service_args = None
        self.browser = webdriver.PhantomJS(service_args=service_args) 

# 需要导入模块: from selenium import webdriver [as 别名]
# 或者: from selenium.webdriver import PhantomJS [as 别名]
def __init__(self, ini, load_image=False, tor_network=False):
        service_args = []
        service_log_path = '/dev/null'

        # if browser don't load image
        if not load_image:
            service_args.append('--load-images=no')

        # selenium connect tor proxy
        if tor_network:
            proxy = '{}:{}'.format(
                ini.read('TOR', 'HOST'),
                ini.read('TOR', 'PORT'))

            service_args.append('--proxy={}'.format(proxy))
            service_args.append('--proxy-type=socks5')

        self.driver = webdriver.PhantomJS(
            executable_path=ini.read('HEADLESS', 'PATH'),
            service_args=service_args,
            service_log_path=service_log_path
        ) 

# 需要导入模块: from selenium import webdriver [as 别名]
# 或者: from selenium.webdriver import PhantomJS [as 别名]
def gethtml(url):
    try:
        headers = {}

        headers['User-Agent'] = 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:21.0) Gecko/20100101 Firefox/21.0'
        headers['Accept-Language'] ='en-us;q=0.5,en;q=0.3'
        headers['Referer'] = url

        u = requests.get(url, timeout=3, headers=headers, allow_redirects=True)
        content = u.text
        return {"html":content,"code":u.status_code,"url":url}

    except Exception as e:
        return get_html_phantomJS(url)

# 使用PhantomJS获取网页源码 

# 需要导入模块: from selenium import webdriver [as 别名]
# 或者: from selenium.webdriver import PhantomJS [as 别名]
def _get_webdriver(self):
        """Return a webdriver instance and set it up
        with the according profile/ proxies.
        Chrome is quite fast, but not as stealthy as PhantomJS.
        Returns:
            The appropriate webdriver mode according to self.browser_type.
            If no webdriver mode could be found, return False.
        """
        if self.browser_type == 'chrome':
            return self._get_Chrome()
        elif self.browser_type == 'firefox':
            return self._get_Firefox()
        elif self.browser_type == 'phantomjs':
            return self._get_PhantomJS()

        return False