Python Browser.click_link_by_partial_text方法代码示例

# 需要导入模块: from splinter import Browser [as 别名]
# 或者: from splinter.Browser import click_link_by_partial_text [as 别名]
class TrackListScraper(object):

    def __init__(self, artists, year):
        self.browser = Browser('chrome')
        self.artists = artists
        self.year = year
        self.browser.visit('http://1001tracklists.com')

    def execute_full_scrape(self):
        artist_tracklists = {}
        for artist in self.artists:
            artist_tracklists[artist] = self.scrape_per_artist(artist)
        self.browser.quit()
        return artist_tracklists

    def scrape_per_artist(self, artist):
        """Execute the same scrape but instead using the python splinter library
        """

        self.browser.fill('main_search', artist + ' edc ' + self.year)

        self.browser.find_by_id('btn_search').first.click()

        try:
            self.browser.click_link_by_partial_text('2014-06-')
            track_strings = self.get_track_list_for_set(artist)
            return track_strings
        except ElementDoesNotExist:
            pass

    def get_track_list_for_set(self, artist):
        soup = BeautifulSoup(self.browser.html)
        track_values = soup.find_all('div', class_='trackValue')

        track_strings = []
        file = open('tracklist-' + artist + '-edc' + self.year, 'w')
        for track in track_values:
            if track.a:
                track_string = track.a.string
                file.write(track_string)
                # track details in format [artist, trackname]
                track_details = self.parse_track_string(track_string)
                track_strings.append(track_details)
        file.close()
        return track_strings

    def parse_track_string(self, track_string):
        track_info = track_string.strip().split('-')
        for i in range(len(track_info)):
            track_info[i] = track_info[i].strip()
        return track_info

# 需要导入模块: from splinter import Browser [as 别名]
# 或者: from splinter.Browser import click_link_by_partial_text [as 别名]
def xfinity(browser=None):
    if not browser:
        print ("Making browser...")
        browser = Browser('phantomjs')
    print ("Trying google.com...")
    browser.visit('http://google.com/')
    if 'google.' in browser.url:
        print ("google.com connected :)")
        return

    print ("Sign up...")
    browser.click_link_by_partial_text('Sign up')
    print ("Filling form...")
    browser.select("rateplanid", "spn")
    browser.check('spn_terms')
    browser.fill('spn_postal', '12345')
    browser.fill('spn_email', '[email protected]')
    print ("Submitting...")
    sleep(3) # it did not work without the sleeps
    browser.find_by_css('.startSessionButton').type(' \n')
    sleep(7)
    browser.ensure_success_response()
    print (browser.screenshot())

# 需要导入模块: from splinter import Browser [as 别名]
# 或者: from splinter.Browser import click_link_by_partial_text [as 别名]
def scrape():
    # NASA Mars News

    # URL of page to be scraped. 
    url = 'https://mars.nasa.gov/news/'

    # Retrieve page with the requests module. Make a request to the url.
    response = requests.get(url)

    # Create a Beautiful Soup object
    soup = BeautifulSoup(response.text, 'html.parser')

    # Latest News Title
    news_title = soup.find('div', class_='content_title').text

    # Get paragraph text
    news_p = soup.find('div', class_='rollover_description_inner').text


    # JPL Mars Space Images - Featured Image

    #chromedriver
    executable_path = {'executable_path': 'chromedriver.exe'}
    browser = Browser('chrome', **executable_path, headless=False)

    # URL of page to be scraped. 
    url = 'https://www.jpl.nasa.gov/spaceimages/?search=&category=Mars'
    browser.visit(url)

    #Feed browser.html into BeautifulSoup
    html = browser.html
    soup = BeautifulSoup(html, 'html.parser')

    browser.click_link_by_partial_text('FULL IMAGE')
    browser.click_link_by_partial_text('more info')

    jpl_html = browser.html
    jpl_soup = BeautifulSoup(jpl_html, 'html.parser')
    temp_url = jpl_soup.find('img', class_='main_image')
    img_url = temp_url.get('src')

    feature_image_url = "https://www.jpl.nasa.gov" + img_url

    #Close the chrome browser 
    browser.quit()             


    #Mars Weather

    # URL of page to be scraped. 
    url = 'https://twitter.com/marswxreport?lang=en'

    # Retrieve page with the requests module. Make a request to the url.
    response = requests.get(url)

    # Create a Beautiful Soup object
    soup = BeautifulSoup(response.text, 'html.parser')

    tweets = []
    tweets = soup.find_all('div', class_="js-tweet-text-container")

    for i in range(20):
        t = tweets[i].text
        if "Sol " in t:
            mars_weather = t
            break


    # Mars Facts

    # URL of page to be scraped. 
    url = 'https://space-facts.com/mars/'

    #List of dataframes of any tables it found
    tables = pd.read_html(url)  

    df = tables[0]
    df.columns = ['Profile','Data']

    #DataFrame to HTML
    html_table = df.to_html()
    mission_to_mars['mars_facts_table'] = html_table
    html_table.replace('\n', '')
    df.to_html('mars_table.html')

    # Mars Hemispheres

    #chromedriver
    executable_path = {'executable_path': 'chromedriver.exe'}
    browser = Browser('chrome', **executable_path, headless=False)

    # URL of page to be scraped. 
    url = 'https://astrogeology.usgs.gov/search/results?q=hemisphere+enhanced&k1=target&v1=Mars'
    browser.visit(url)

    html = browser.html
    soup = BeautifulSoup(html, 'html.parser')    

    hemisphere_image_urls = []                    

#.........这里部分代码省略.........

# 需要导入模块: from splinter import Browser [as 别名]
# 或者: from splinter.Browser import click_link_by_partial_text [as 别名]
def scrape():

    executable_path = {'executable_path': 'chromedriver.exe'}
    browser = Browser('chrome', **executable_path, headless=False)


    # # In[9]:


    url = 'https://www.jpl.nasa.gov/spaceimages/?search=&category=Mars'
    browser.visit(url)


    # # In[10]:


    html = browser.html
    soup = bs(html, 'html.parser')
    browser.click_link_by_partial_text('FULL IMAGE')


    # # In[11]:


    # #needs a pause or else code runs too fast
    time.sleep(2)
    browser.click_link_by_partial_text('more info')


    # # In[12]:


    html2 = browser.html
    soup2 = bs(html2, 'html.parser')
    image = soup2.find('img', class_='main_image')


    url = image.get('src')

    featured_image_url = 'https://www.jpl.nasa.gov' + url
    # #print(featured_image_url)
    time.sleep(2)
    browser.quit()


    # # In[13]:


    # #Visit the Mars Weather twitter account here and scrape the latest Mars weather tweet from the page. Save the tweet text for the weather report as a variable called mars_weather.
    url = 'https://twitter.com/marswxreport?lang=en'
    response = requests.get(url)
    soup = bs(response.text, 'html.parser')
    # #print(soup.prettify())


    # # In[14]:


    results = soup.find_all('div', class_='js-tweet-text-container')
    # #print(results)


    # # In[15]:


    mars_tweet= results[0].text
    # #print(mars_tweet)


    # # In[16]:


    # #Visit the Mars Facts webpage here and use Pandas to scrape the table containing facts about the planet including Diameter, Mass, etc.
    # #Use Pandas to convert the data to a HTML table string.
    mars_facts_url = 'https://space-facts.com/mars/'


    # # In[17]:


    tables = pd.read_html(url)
    tables


    # # In[18]:


    df = tables[0]
    df.head()


    # # In[19]:


    df.set_index(0, inplace=True)
    clean_df = df
    clean_df


    # # In[20]:
#.........这里部分代码省略.........

# 需要导入模块: from splinter import Browser [as 别名]
# 或者: from splinter.Browser import click_link_by_partial_text [as 别名]

#.........这里部分代码省略.........
    	path = self.__browser.driver.firefox_profile.profile_dir
    	print path
    	os.remove(constants.profile+'/places.sqlite')
    	shutil.copyfile(path+'/places.sqlite', constants.profile+'/places.sqlite')
        self.__closeWindow()
    	shutil.rmtree(path)
    	#os.rmdir(path)
        print "Firefox beendet"
        
        
    def starte(self):
        self.run()
    
    def __generateRandom(self):
        self.toWait = random.randrange(5,45)
        self.elemNo = random.randrange(0,len(self.seiten))
        self.clickNo = random.randrange(2,7)
        self.back = random.randrange(0,10)
        self.wordNo = random.randrange(0, len(self.words))
    
    def __generateRandomClick(self):
        self.clickX = random.randrange(100,constants.BREITE - 50) #1366
        self.clickY = random.randrange(50,constants.HOEHE-50) #768
        command = "mousemove "+ str(self.clickX) + " "+ str(self.clickY)
        print command
        subprocess.call(["xte", command])
        subprocess.call(["xte", "mouseclick 1"])
      
    def __followLink(self, text, index=0):
        if index == None:
            index = 0
        
        try:   
            self.__browser.click_link_by_partial_text(text)[index]
        except ElementDoesNotExist:
            print "Element does not exist"
        except TypeError:
            print "Type Error"
        except Exception as e: 
	       print "nix passiert" + e
    
    def __visitGooglePage(self, url):
             
        print "google"
        
        self.__browser.visit(url)
        time.sleep(random.randrange(2,15))
        searchWord = str(self.words[self.wordNo]).strip().decode("utf-8")
        print searchWord
        self.__fillInput('q', searchWord)
        time.sleep(random.randrange(2,15))
        self.__findElementAndClick("btnG", "name", None)
        subprocess.call(["xte", "key Return"])
        wordSplit = str(searchWord).split(" ")
        time.sleep(random.randrange(10,30))
            #baaaad practice
        try:
            self.__followLink(wordSplit[0], self.wordNo%10)
        except Exception:
            try: 
                self.__followLink(wordSplit[1], self.wordNo%10)
            except Exception:
                    pass
        
        
    def __visitHomepage(self, url):

# 需要导入模块: from splinter import Browser [as 别名]
# 或者: from splinter.Browser import click_link_by_partial_text [as 别名]
browser.visit('http://www.baidu.com')
print browser.url
print browser.title
print browser.html

# Input search text
browser.fill('wd', '12306')

# Press the search button
button = browser.find_by_id('su')
button.click()

# Interacting with elements in the page
# (the find_* method returns a list of all found elements)
# (If an element is not found, the find_* methods return an empty list.
#  But if you try to access an element in this list,
#  the method will raise splinter.exceptions.ElementDoesNotExist )

# [1] Get value of an element
content_left = browser.find_by_id('content_left')
print len(content_left)
print content_left[0].value

# [2] Clicking links
browser.click_link_by_partial_text(u'铁道部火车票网上订票唯一官网 - 铁路客户服务中心')

# Close the browser
import time
time.sleep(10)
browser.quit()