Python Firefox.find_element_by_xpath方法代码示例

# 需要导入模块: from selenium.webdriver import Firefox [as 别名]
# 或者: from selenium.webdriver.Firefox import find_element_by_xpath [as 别名]
def ff():
    browser = Firefox()
    browser.get('https://www.charter.com/browse/content/new-channel-lineup')
    wait = WebDriverWait('browser', 20)

    button = browser.find_element_by_xpath('//*[@id="select-state"]/div[2]') # find and click Choose State button
    button.click()

# 需要导入模块: from selenium.webdriver import Firefox [as 别名]
# 或者: from selenium.webdriver.Firefox import find_element_by_xpath [as 别名]
class CaptchaPage():
    def __init__(self):
        print "Captcha Page Initializing"

        parser = ConfigParser.ConfigParser()
        base_path = os.path.join(os.environ['HOME'], '.mozilla/firefox/')
        parser.read(os.path.join(base_path, "profiles.ini"))
        profile_path = os.path.join(base_path, filter(lambda x: x[0].lower() == 'path', parser.items('Profile0'))[0][1])
        try:
            profile = FirefoxProfile(profile_path)
        except OSError:
            raise Exception("You must execute the following command:\nsudo chmod +r -R %s" % profile_path)
        self.driver = Firefox(profile)

        self.driver.get("file://%s/index.html" % os.getcwdu())

    def get_url_sound(self):
        self.driver.find_element_by_xpath('//*[@id="recaptcha_switch_audio"]').click()
        return self.driver.find_element_by_xpath('//*[@id="recaptcha_audio_download"]').get_attribute('href')

    def get_recaptcha_challenge_field(self):
        return self.driver.find_element_by_xpath('//*[@id="recaptcha_challenge_field"]').get_attribute('value')

    def get_captcha_textbox(self):
        print "Getting Captcha Textbox"
        return Textbox(self.driver.find_element_by_xpath('//*[@id="recaptcha_response_field"]'))

    def get_submit_button(self):
        print "Getting Submit Form Button"
        return Button(self.driver.find_element_by_xpath("/html/body/form/input"))

    def close(self):
        print "Closing Captcha Page"
        self.driver.close()

# 需要导入模块: from selenium.webdriver import Firefox [as 别名]
# 或者: from selenium.webdriver.Firefox import find_element_by_xpath [as 别名]
def login(username, password):
  """Login into website, return cookies, api and sso token using geckodriver/firefox headless"""

  display = Display(visible=0, size=(800, 600))
  display.start()
#  options = Options()
#  options.add_argument('-headless')
#  driver = Firefox(executable_path='/usr/local/bin/geckodriver', firefox_options=options)
  driver = Firefox()
  wait = WebDriverWait(driver, timeout=10)

  driver.get(url)
  time.sleep(10)

  username_field = driver.find_element_by_name("emailOrPcrNumber")
#  There are multiple entries with the name pin, use the xpath instead even though it is more error prone
#  password_field = driver.find_element_by_name("pin")
  password_field = driver.find_element_by_xpath('/html/body/div[1]/div/div/div[2]/div[1]/div[2]/form/div/div[1]/div[2]/input')

  username_field.clear()
  username_field.send_keys(username)

  password_field.clear()
  password_field.send_keys(password)

  time.sleep(2)
  driver.find_element_by_id("tpiSubmitButton").click()

  time.sleep(3)
  cookies = driver.get_cookies()
  for cookie in cookies:
    if cookie['name'] == 'X-IHG-SSO-TOKEN':
      sso_token = cookie['value']
  api_key = driver.execute_script('return AppConfig.featureToggle.apiKey')

  driver.get('https://apis.ihg.com')
  cookies.extend(driver.get_cookies())
  driver.quit()
  display.stop()
  return api_key, sso_token, cookies

# 需要导入模块: from selenium.webdriver import Firefox [as 别名]
# 或者: from selenium.webdriver.Firefox import find_element_by_xpath [as 别名]
    find_st = browser.find_element_by_link_text(st_prompt.upper())
    sel_st = find_st.text + ', USA'
    print sel_st
    find_st.click() # click to sel state
    sleep(1)
    try:
        list_region = WebDriverWait(browser, 02).until(EC.visibility_of((By.ID, "select-region")))
        # list_region = browser.find_element_by_xpath('//*[@id="select-region"]/div[2]') # find Choose Region button
        list_region.click()
        print "Success!"
        break
    else:
        print "No Charter!"

print "Moving on..."
cur_state = browser.find_element_by_xpath('//*[@id="select-region"]/div[3]')







# text_region = cur_state.find_elements_by_tag_name('a') # find all regions by <a> tag
def temp():
    l_regions = cur_state.text

    print "l_regions: "
    print l_regions
    # creates text file with all the region
    regions = open('Regions.text', 'w+')

# 需要导入模块: from selenium.webdriver import Firefox [as 别名]
# 或者: from selenium.webdriver.Firefox import find_element_by_xpath [as 别名]
browser.get('https://www.charter.com/browse/content/new-channel-lineup')

# Click the "choose state" menu using Selenium
wait = WebDriverWait('browser', 20)
button = browser.find_element_by_css_selector('.drop-down-current')
button.click()

# # Identify all states in the list, read as text using Selenium
list_item = browser.find_element_by_class_name('drop-down-list')
states = list_item.text

# sel_st = raw_input('Type in 2 letter st abbreviation: ')
find_st = browser.find_element_by_link_text('CO')
find_st.click()
sleep(1)
list_region = browser.find_element_by_xpath('//*[@id="select-region"]/div[2]') # find and click Choose Region button
list_region.click()
cur_state = browser.find_element_by_xpath('//*[@id="select-region"]/div[3]')

# text_region = cur_state.find_elements_by_tag_name('a') # find all regions by <a> tag
l_regions = cur_state.text

# creates text file with all the region
regions = open('Regions.text', 'w')
regions.write(l_regions)

print l_regions
# Place region points on map
gmaps2.gmaps_mapit(l_regions)

# 需要导入模块: from selenium.webdriver import Firefox [as 别名]
# 或者: from selenium.webdriver.Firefox import find_element_by_xpath [as 别名]
class BaiDuWaiMaiCrawler:
    def __init__(self):
        self.comment_root_url = "http://waimai.baidu.com/shopui/?qt=shopcomment&shop_id="
        self.comment_root_path = "files/baiduwaimai_comments-%s.json" % datetime.now().strftime("%Y-%m-%d")
        self.browser = Firefox()
        self.ids = defaultdict(list)
        self.crawled_ids = []
        self.crawled_id_filepath = "files/crawled_ids.txt"
        self.get_crawled_ids()

    def __del__(self):
        self.browser.quit()

    def get_crawled_ids(self):
        if exists(self.crawled_id_filepath):
            with open(self.crawled_id_filepath, encoding="utf-8") as f:
                for line in f:
                    self.crawled_ids.append(line.strip())

    def record_crawled_id(self, shop_id):
        with open(self.crawled_id_filepath, mode="a", encoding="utf-8") as f:
            f.write("%s\n" % shop_id)

    @staticmethod
    def get_address_urls_from_file():
        urls = []
        pattern = re.compile("\s+")
        with open("files/baiduwaimai_address_urls.txt") as f:
            for line in f:
                results = pattern.split(line.strip())
                if len(results) >= 2:
                    urls.append(results[0])
        print("从文件内得到所有地址的url")
        return urls

    def get_shop_ids_from_file(self, filepath, encoding="utf-8"):
        pattern = re.compile("\s+")
        with open(filepath, encoding=encoding) as f:
            for line in f:
                results = pattern.split(line.strip())
                if len(results) >= 2:
                    self.ids[results[0]] = results[1].split(",")

    def get_shop_ids_from_net(self):
        address_urls = self.get_address_urls_from_file()
        for index, url in enumerate(address_urls):
            self.shop_urls_at_a_address(url, index)

    def shop_urls_at_a_address(self, url, line_index):
        self.browser.get(url)
        self.browser.maximize_window()
        for i in range(10):
            self.browser.find_element_by_id("baiducopy").click()
            time.sleep(2)
        page_source = self.browser.page_source
        # self.browser.close()

        soup = BeautifulSoup(page_source, "html.parser")
        if soup.find("ul", class_="shopcards-list"):
            for li in soup.find("ul", class_="shopcards-list").find_all("li", class_="list-item"):
                key = li.get("class")[2][4:]
                address_id = str(line_index)
                self.ids[key].append(address_id)

    def get_comments_in_one_shop(self, shop_id):
        self.browser.get("%s%s" % (self.comment_root_url, shop_id))
        self.browser.maximize_window()
        while True:
            footer = self.browser.find_element_by_xpath("//div[@class='footer-items']")
            for i in range(2):
                ActionChains(self.browser).move_to_element(footer).perform()
                time.sleep(1)

            page_source = self.browser.page_source
            soup = BeautifulSoup(page_source, "html.parser")
            div = soup.find("section", "comment-list").find("div", "comment-con")
            if div.find("div", class_="no-result") is not None:
                break
            else:
                for a_div in div.find_all("div", class_="list clearfix"):
                    self.get_one_comment(a_div, shop_id)
            try:
                the_next = self.browser.find_element_by_xpath(
                    "//div[@class='pagination']//a[@class='mod-page-item mod-page-item-next']")
                the_next.click()
                time.sleep(2)
            except NoSuchElementException:
                break
        # self.browser.close()
        print("爬完ID为 '", shop_id, "' 的餐厅的评论信息。")
        self.record_crawled_id(shop_id)
        self.crawled_ids.append(shop_id)

    def get_one_comment(self, div, shop_id):
        try:
            comment_info = {"shop_id": shop_id}
            top_sec = div.find("div", class_="top-section").get_text("|", strip=True).split("|")
            comment_info["user_name"] = top_sec[0]  # a_div.find("span", class_="user-name").string.strip()
            comment_info["mark"] = top_sec[1][:-1]
            comment_info["delivery_time"] = top_sec[2]  # a_div.find("span", class_="delivery-time").string.strip()
#.........这里部分代码省略.........

# 需要导入模块: from selenium.webdriver import Firefox [as 别名]
# 或者: from selenium.webdriver.Firefox import find_element_by_xpath [as 别名]
class TestMaxlifeFeature(object):
    """
    Checks if the maxlife feature is working
    """

    def setup_class(self):
        """
        Setup: Open a mozilla browser, login
        """
        self.browser = Firefox()
        self.browser.get('http://localhost:5000/')
        token = self.browser.find_element_by_name("token")
        password = "foo"
        # login
        token.send_keys(password)
        token.send_keys(Keys.ENTER)
        time.sleep(.1)
        try:
            self.browser.find_element_by_xpath("//input[@value='Logout']")
        except NoSuchElementException:
            raise ValueError("Can't login!!! Create a user 'foo' with the permissions"
                             "'read' and 'write' in your PERMISSIONS in the config")

    def teardown_class(self):
        """
        Tear down: Close the browser
        """
        self.browser.quit()

    def test_unit_input_exists(self):
        unit_input = self.browser.find_element_by_name("maxlife-unit")
        assert unit_input is not None
        value_input = self.browser.find_element_by_name("maxlife-value")
        assert value_input is not None

    def fill_form(self):
        paste_input = self.browser.find_element_by_id("formupload")
        paste_input.send_keys("This is test")
        filename_input = self.browser.find_element_by_id("filename")
        filename_input.send_keys("test.txt")
        contenttype_input = self.browser.find_element_by_id("contenttype")
        contenttype_input.send_keys("text/plain")
        contenttype_input.send_keys(Keys.ENTER)

    def delete_current_file(self):
        self.browser.find_element_by_id("del-btn").click()
        time.sleep(.2)
        self.browser.find_element_by_class_name("btn-primary").click()

    def test_paste_keep_forever(self):
        self.browser.find_element_by_xpath("//select[@name='maxlife-unit']/option[@value='forever']").click()
        value_input = self.browser.find_element_by_name("maxlife-value")
        value_input.clear()
        value_input.send_keys(1)
        self.fill_form()
        assert "max life" not in self.browser.find_element_by_tag_name("body").text.lower()
        self.delete_current_file()

    def test_paste_keep_minutes(self):
        self.browser.find_element_by_xpath("//select[@name='maxlife-unit']/option[@value='minutes']").click()
        value_input = self.browser.find_element_by_name("maxlife-value")
        value_input.clear()
        value_input.send_keys(1)
        self.fill_form()
        assert "max time" in self.browser.find_element_by_tag_name("body").text.lower()
        self.delete_current_file()

    @pytest.mark.slow
    def test_file_gets_deleted(self):
        self.browser.find_element_by_xpath("//select[@name='maxlife-unit']/option[@value='minutes']").click()
        value_input = self.browser.find_element_by_name("maxlife-value")
        value_input.clear()
        value_input.send_keys(1)
        self.fill_form()
        time.sleep(61)
        self.browser.find_element_by_id("inline-btn").click()
        assert "not found" in self.browser.find_element_by_tag_name("body").text.lower()

# 需要导入模块: from selenium.webdriver import Firefox [as 别名]
# 或者: from selenium.webdriver.Firefox import find_element_by_xpath [as 别名]
class Hertz():
	def __init__( self ):
		self.driver = Firefox()
		#self.driver.get( 'https://www.hertzequip.com/herc/rental-equipment/aerial-equipment+manlifts+articulating-boom-lift' )
		self.driver.implicitly_wait( 15 )
		
	def perform_search( self, zip_code ):
		self.driver.get( 'https://www.hertzequip.com/herc/rental-equipment/aerial-equipment+manlifts+articulating-boom-lift' )
		#  wait for javascript to load the page before we begin
		self.driver.find_element_by_xpath( '//span[text()="View Rates"]' )
		#  selenium commands can be very unreliable
		#  since Jquery is already loaded, I'll use that
		self.driver.execute_script( """
			$('span:contains("View Rates")')[0].click()
			$('input[id="postalCode"]').val('""" + zip_code + """')
			$('a[class="button_four button_four_skin"]')[0].click()
			""" )
		#  wait a few seconds for page element to disappear
		sleep( 3 )
		#  wait until javascript has loaded the page again
		self.driver.find_element_by_xpath( '//span[text()="View Rates"]' )
			
	def get_rates( self ):
		rates_info_json = self.driver.execute_script( """	
			var xmlhttp = new XMLHttpRequest()
			var url = "https://www.hertzequip.com/herc/rest/rentals/getBuilderEquipmentDetailsItemInfo?catalogType=" + String( catalogType ) + "&categoryId=" + categoryId + "&subcategory1Id=" + subcategory1Id + "&subcategory2Id=" + subcategory2Id + "&_=" + String( new Date().getTime() )
			xmlhttp.open( "GET", url, false )
			xmlhttp.send()
			return xmlhttp.responseText
			""" )
		return rates_info_json
			
	def get_store_info( self ):
		store_info_json = self.driver.execute_script( """
			var xmlhttp = new XMLHttpRequest()
			var url = "https://www.hertzequip.com/herc/rest/rentals/getBuilderJobLocationsItemInfo?_=" + String( new Date().getTime() )
			xmlhttp.open( "GET", url, false )
			xmlhttp.send()
			return xmlhttp.responseText
			""" )
		return store_info_json
		
	def perform_search_get_data( self, zip_code ):
		self.perform_search( zip_code )
		location_data = json.loads( self.get_store_info() )
		equipment_rates_data = json.loads( self.get_rates() )
		specific_data = OrderedDict()
		specific_data['branch_zip_code'] = location_data['data']['jobLocation']['branchLocation']['zip']
		specific_data['branch_id'] = location_data['data']['jobLocation']['branchLocation']['branch']
		specific_data['equipment_list'] = []
		for equipment in equipment_rates_data['data']['equipmentInfo']['m_list']:
			equipment_data = OrderedDict()
			equipment_data['specs'] = [ equipment['spec1'], equipment['spec2'] ]
			equipment_data['makes & models'] = equipment['makesModels']['m_list']
			equipment_data['rates'] = equipment['ratesInfo']
			specific_data['equipment_list'].append( equipment_data )
		return { 
			'location_data' : location_data,
			'equipment_rates_data' : equipment_rates_data,
			'specific_data' : specific_data
			}
		
		
		

# 需要导入模块: from selenium.webdriver import Firefox [as 别名]
# 或者: from selenium.webdriver.Firefox import find_element_by_xpath [as 别名]
class RegistrationWebTest(TestCase):
    """
    Test all facets of the registration process
    """

    @classmethod
    def clear_database(cls):
        """
        Clear the database before and after use
        """
        collection = cls.mongo.collection
        for user in ['UnittestExistingTestUser', 'UnittestNonExistingTestUser']:
            test_user = collection.find_one({
                'username': user,
            })
            if test_user:
                collection.remove(test_user)

    @classmethod
    def setUpClass(cls):
        """
        Setup test data, browser and server
        """
        cls.mongo = UserDatabaseConnectivity()
        cls.clear_database()
        test_user = {
            'username': 'UnittestExistingTestUser',
            'salt': '000',
            'password': '000',
            'enabled': False,
        }
        cls.mongo.collection.save(test_user)
        cls.config = dict()
        prepare_test(cls)
        cls.base_url = 'http://{:s}:{:d}/static/index.xhtml'.format(cls.config['bind_ip'], cls.config['bind_port'])

    @classmethod
    def tearDownClass(cls):
        """
        Disconnect from mongo and cleanup browser, server, etc.
        """
        cls.clear_database()
        del cls.mongo
        cleanup(cls)

    def setUp(self):
        """
        Force a page refresh between tests
        """
        self.webdriver = Firefox()
        self.webdriver.implicitly_wait(10)

    def tearDown(self):
        """
        Throw test user out of database
        """
        self.webdriver.close()
        self.webdriver.quit()
        collection = self.mongo.collection
        test_user = collection.find_one({
            'username': 'UnittestNonExistingTestUser',
        })
        if test_user:
            collection.remove(test_user)

    def __util_get_reg_button(self):
        """
        Get the registration form button
        """
        self.webdriver.get(self.base_url)
        self.webdriver.implicitly_wait(10)
        sleep(3)
        button = self.webdriver.find_element_by_xpath('//xhtml:button[@data-formaction="registrationForm"]')
        return button

    def __util_open_dialog(self):
        """
        Open the registration dialog
        """
        button = self.__util_get_reg_button()
        button.click()
        self.webdriver.implicitly_wait(10)
        sleep(5)

    def test_find_button(self):
        """
        Is the button there?
        """
        self.assertIsNotNone(self.__util_get_reg_button())

    def test_open_dialog(self):
        """
        Can we open the dialog?
        """
        dialog_xpath = '//xhtml:div[contains(@class, "bootstrap-dialog")]'
        # Test that there is no dialog open at the moment
        self.assertRaises(NoSuchElementException, self.webdriver.find_element_by_xpath, dialog_xpath)
        self.__util_open_dialog()
        dialog = self.webdriver.find_element_by_xpath(dialog_xpath)
        self.assertIsNotNone(dialog)
#.........这里部分代码省略.........

# 需要导入模块: from selenium.webdriver import Firefox [as 别名]
# 或者: from selenium.webdriver.Firefox import find_element_by_xpath [as 别名]

#.........这里部分代码省略.........
        except:
            property_data['beds'] = 'N/A';print('beds not found')
        try:
            property_data['baths'] = self.soup.find('div', attrs={'data-rf-test-id': 'abp-baths'}).find(
                'div').get_text()
        except:
            property_data['baths'] = 'N/A';print('baths not found')
        try:
            property_data['sqFt'] = self.soup.find('div', attrs={'data-rf-test-id': 'abp-sqFt'}).find('span', attrs={
                'class': 'main-font statsValue'}).get_text()
        except:
            property_data['sqFt'] = 'N/A';print('sqFt not found')
        try:
            property_data['price_per_sqFt'] = self.soup.find('div', attrs={'data-rf-test-id': 'abp-sqFt'}).find('div',
                                                                                                                attrs={
                                                                                                                    "data-rf-test-id": "abp-priceperft"}).get_text()
        except:
            property_data['price_per_sqFt'] = 'N/A';print('price_per_sqFt not found')
        try:
            property_data['year_built'] = self.soup.find('span', attrs={"data-rf-test-id": "abp-yearBuilt"}).find(
                'span', attrs={'class': 'value'}).get_text()
        except:
            property_data['year_built'] = 'N/A';print('year_built not found')
        try:
            property_data['days_on_redfin'] = self.soup.find('span',
                                                             attrs={"data-rf-test-id": "abp-daysOnRedfin"}).find('span',
                                                                                                                 attrs={
                                                                                                                     'class': 'value'}).get_text()
        except:
            property_data['days_on_redfin'] = 'N/A';print('days_on_redfin not found')
        try:
            property_data['status'] = self.soup.find('span', attrs={"data-rf-test-id": "abp-status"}).find('span',
                                                                                                           attrs={
                                                                                                               'class': 'value'}).get_text()
        except:
            property_data['status'] = 'N/A';print('status not found')

        property_data['summary'] = self.soup.find('div', attrs={'class': 'remarks'}).get_text()
        for row in self.soup.find('div', attrs={'class': 'more-info-div'}).find_all('tr'):
            cells = row.find_all('td')
            property_data[cells[0].get_text().strip()] = cells[1].get_text().strip()

        # use loops to maintain data structure ina dict
        property_data['property_details'] = OrderedDict()
        for category in self.soup.find('div', attrs={'class': 'amenities-container'}).children:
            key = category.contents[0].get_text().strip()
            property_data['property_details'][key] = OrderedDict()
            for row in category.contents[1].find_all('div', attrs={'class': 'amenity-group'}):
                key2 = row.find('h4').get_text()
                property_data['property_details'][key][key2] = []
                for row2 in row.find_all('li'):
                    property_data['property_details'][key][key2].append(row2.get_text())

        property_data['propert_history'] = []
        for row in self.soup.find_all('tr', attrs={'id': reg_property_history_row}):
            data_cells = row.find_all('td')
            history_data_row = OrderedDict()
            history_data_row['date'] = data_cells[0].get_text()
            history_data_row['event & source'] = data_cells[1].get_text()
            history_data_row['price'] = data_cells[2].get_text()
            history_data_row['appreciation'] = data_cells[3].get_text()
            property_data['propert_history'].append(history_data_row)

        property_data['url'] = 'https://www.redfin.com' + property_url
        self.output_data.append(property_data)
        return property_data

    def use_browser(self):
        self.use_selenium = True
        firefox_profile = FirefoxProfile()
        #  might as well turn off images since we don't need them
        if self.use_proxies:
            #  if use proxies is true load firefox with proxies
            firefox_profile.set_preference("permissions.default.image", 2)
            proxy_host, proxy_port = choice(self.proxies).split(':')
            firefox_profile.set_preference("network.proxy.type", 1)
            firefox_profile.set_preference("network.proxy.http", proxy_host)
            firefox_profile.set_preference("network.proxy.http_port", int(proxy_port))
            firefox_profile.set_preference("network.proxy.ssl", proxy_host)
            firefox_profile.set_preference("network.proxy.ssl_port", int(proxy_port))
        self.driver = Firefox(firefox_profile)
        self.driver.implicitly_wait(2)

    def get_page_selenium(self, page_url):
        self.driver.get(page_url)
        self.selenium_bypass_captcha()
        return self.driver.page_source

    def selenium_bypass_captcha(self):
        #  basic code for handling captcha
        #  this requires the user to actually solve the captcha and then continue
        try:
            self.driver.switch_to_frame(self.driver.find_element_by_xpath('//iframe[@title="recaptcha widget"]'))
            self.driver.find_element_by_class_name('recaptcha-checkbox-checkmark').click()
            print('solve captcha ( pop up only ) and press enter to continue')
            raw_input()
            self.driver.switch_to_default_content()
            self.driver.find_element_by_id('submit').click()
        except Exception as e:
            pass