Python Ghost.wait_for_page_loaded方法代码示例

# 需要导入模块: from ghost import Ghost [as 别名]
# 或者: from ghost.Ghost import wait_for_page_loaded [as 别名]
class IDirectBroker(object):
    
    IDIRECTURL = 'https://secure.icicidirect.com/Trading/LBS/Logon.asp'

    def __init__(self,username, password, **kwargs):
        self.username = username
        self.password = password
        self.ghost = Ghost()



        try:
            self.page, self.resources = self.ghost.open(self.IDIRECTURL)
            
            self.ghost.wait_for_page_loaded()
            self.ghost.capture_to("./l1.png")
            
            result, resources = self.ghost.fill("form", { "FML_USR_ID": "MUSE9L71", "FML_USR_USR_PSSWRD": "[email protected]","FML_USR_DT_BRTH":"22101982" })
            self.ghost.capture_to("./l2.png")
            self.page, self.resources = self.ghost.fire("form", "submit", expect_loading=True)
            #self.ghost.wait_for_page_loaded()
            self.ghost.capture_to("./l3.png")

        except Exception,e:
            raise e

# 需要导入模块: from ghost import Ghost [as 别名]
# 或者: from ghost.Ghost import wait_for_page_loaded [as 别名]
def ghost_img(src):
	ghost = Ghost()
	page,resources = ghost.open(src)
	ghost.wait_for_page_loaded()  
	result, resources = ghost.evaluate("document.getElementById('largeImg').getAttribute('src');" ) 
	del resources
	return result

# 需要导入模块: from ghost import Ghost [as 别名]
# 或者: from ghost.Ghost import wait_for_page_loaded [as 别名]
def get_excede_html():
	result = 'unknown'
	usage_page_html = ''
	username, password = load_auth_info()
	#print "ghost initialized..."
	ghost = Ghost(wait_timeout=50, log_level='DEBUG', download_images=False)
	
	#Check for FAIL: no internet access
	try:
		page,resources = ghost.open('https://my.exede.net/usage')
	except Exception as e:
		print e.message
		result = e.message
		ghost.exit()
		return usage_page_html, result

	ghost.wait_for_page_loaded()

	if ghost.exists(".form-control.input-lg.required[name=IDToken1]"):
		print "Login found"
	else:
		print "Login not found"
		result = 'Can\'t find login box on website'
		ghost.exit()
		return usage_page_html, result

	print "Filling in field values"
	ghost.set_field_value(".form-control.input-lg.required[name=IDToken1]",username)
	ghost.set_field_value(".form-control.input-lg.required[name=IDToken2]",password)
	print "Clicking form button"
	ghost.click('.btn.btn-info.btn-lg.pull-right.col-lg-4[name="Login.Submit"]')
	print "Waiting for page to load"
	ghost.wait_for_page_loaded()

	try:
		if ghost.wait_for_selector('.amount-used',timeout=60):
			print "Found the amount used..."
			result = 'OK'
		else:
			print "Did not find the amount used"	
			result = 'OK'
			#ghost.exit()
			#return usage_page_html, result

	except Exception as e:
		print e.message
		result = e.message

	
	usage_page_html = ghost.content.encode('ascii', 'ignore')
	
	print "Writing resulting page to disk as final_page.html"
	with open('final_page.html', 'w') as data_file:
		data_file.write(usage_page_html)
	
	ghost.exit()
	return usage_page_html, result

# 需要导入模块: from ghost import Ghost [as 别名]
# 或者: from ghost.Ghost import wait_for_page_loaded [as 别名]
class GAGetter(object):

    def __init__(self, email, password):
        self.email = email
        self.password = password
        self.ghost = Ghost(wait_timeout=20)

    def test(self):
        self.sign_in()
        self.go_to_realtime_site()
        sleep(10)
        print 'COUNTER: {0}'.format(self.get_counter())

    def sign_in(self):
        page, resources = self.ghost.open(
                                'http://www.google.com/analytics/index.html')
        self.ghost.wait_for_text('Analytics')
        self.ghost.wait_for_selector('a.secondary-button')
        self.ghost.click('a.secondary-button')
        self.ghost.wait_for_text("Can't access your account?")
        self._fill_signin_form()
        self.ghost.wait_for_text('All Accounts')

    def _fill_signin_form(self):
        result, resources = self.ghost.fill(
                    "form", {
                        "Email": self.email,
                        "Passwd": self.password
                    }
                                            )

        page, resources = self.ghost.fire_on(
                                "form", "submit", expect_loading=True)
        self.ghost.wait_for_page_loaded()

    def go_to_realtime_site(self):
        m = re.search(
            r"#report/visitors-overview/([a-z0-9]+)/",
            self.ghost.content
        )
        self.ghost.evaluate(
            "window.location.href = '#realtime/rt-overview/{0}/'".format(
                m.group(1)
            )
        )
        self.ghost.wait_for_text('Right now')
        return self.ghost.content

    def get_counter(self):
        d = pq(self.ghost.content)
        return d('#ID-overviewCounterValue').html()

# 需要导入模块: from ghost import Ghost [as 别名]
# 或者: from ghost.Ghost import wait_for_page_loaded [as 别名]
def make_pages(pn, c):
         
    #global ghost
    ghost = Ghost(download_images = True)

    #time.sleep( 1 )

    root = app_root + '/cache'
    file = "page_" + pn + "_" + str(c) +  ".jpg"

    markup = get_markup(pn, True)

    ghost.main_frame.setHtml(markup)
    ghost.wait_for_page_loaded()
    ghost.capture_to(root + '/' + file)

# 需要导入模块: from ghost import Ghost [as 别名]
# 或者: from ghost.Ghost import wait_for_page_loaded [as 别名]
def get_disqus_comments_by_ghost(dictionary):
    '''Uses Ghost to trigger url for iframe,
    then requests to fetch that html,
    and finally get the json from failed-page from disqus.. 
    '''
    ghost = Ghost(user_agent='Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:25.0) Gecko/20100101 Firefox/25.0', viewport_size = (1349, 765), log_level=logging.ERROR)
    # user_agent='Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_0 like Mac OS X; en-us) AppleWebKit/532.9 (KHTML, like Gecko) Version/4.0.5 Mobile/8A293 Safari/6531.22.7', viewport_size = (320, 480) 
    page, resources = ghost.open(dictionary['url'])
    assert page.http_status == 200      # make use we get data back..
    ghost.wait_for_page_loaded()        # probably does no harm..

    # comment loaded on scroll hack cred goes to Hammer et al. (2013)
    secs = 0.50
    time.sleep(secs)
    ghost.evaluate("window.scroll(0, 700);")
    ghost.capture_to('scroll_before.png')       # do not get why this fails if i remove this image-capture function...
    time.sleep(secs)
    ghost.evaluate("window.scroll(0, 1400);")
    time.sleep(secs)
    ghost.evaluate("window.scroll(0, 2100);")
    time.sleep(secs)
    ghost.evaluate("window.scroll(0, 4000);")
    time.sleep(secs)
    ghost.wait_for_page_loaded()                #ghost.capture_to('scroll_after.png')
    logger.info("waiting for selector IFRAME")
    ghost.wait_for_selector("iframe") ##post-list

#    print ghost.content
    soup = BeautifulSoup(ghost.content)
    try:
        comments_iframe_url = soup.select("iframe#dsq-2")[0]['src'] # only one of these...
        # headers = {
        #     'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:25.0) Gecko/20100101 Firefox/25.0',
        #     'X-UA-Compatible': 'IE=edge'
        # }
        comments_html = requests.get(comments_iframe_url) # , headers=headers
        #print comments_html.text
        iframe_soup = BeautifulSoup(comments_html.text)
        posts = iframe_soup.select("#disqus-threadData")
        data = json.loads(posts[0].text)
        
        #print type(data)
        return data['response']['thread']['posts']
    except:
        # fetching comments failed
        return -9999

# 需要导入模块: from ghost import Ghost [as 别名]
# 或者: from ghost.Ghost import wait_for_page_loaded [as 别名]
class KuKlueCrawler:
	# Class 생성자, id, pw로 KuKlue 로그인 시도
	def __init__(self, id, pw, displayFlag = False, download_images=False, prevent_download=["css"]):
		# 새 Ghost instance를 만들어서 사용합니다.
		self.ghost = Ghost(display = displayFlag, wait_timeout = 60)
		self.currentPage = None
		self.login(id, pw)

	# Class 소멸자
	def __del__(self):
		self.ghost.exit()
		del self.ghost

	# 요청한 url 접속
	def openPage(self, url):
		page, resource = self.ghost.open(url)
		self.ghost.wait_for_page_loaded()
		self.currentPage = url
		return page, resource

	# Login 하는 함수
	def login(self, id, pw):
		page, resource = self.openPage('http://klue.kr/')

		self.ghost.evaluate("""
		(function() {        
		  document.getElementById('mb_id').value = '%s';
		  document.getElementById('mb_pw').value = '%s';
		  document.getElementsByClassName('login')[0].click();
		})();
		""" % (id, pw), expect_loading = True)
 		return page, resource

 	# 특정 index에 대해 lecture page 접속
	def main_search(self, query=None, lectureNum=-1):
		if query is not None: 
			self.ghost.wait_for_selector('#topBar_search')

			self.ghost.fill("#sform", { "query": query })
			page, resource = self.ghost.fire_on('#sform', 'submit', expect_loading = True)
			
		if lectureNum != -1:
			page, resource = self.openPage('http://klue.kr/lecture.php?no='+str(lectureNum))

		return page, resource

# 需要导入模块: from ghost import Ghost [as 别名]
# 或者: from ghost.Ghost import wait_for_page_loaded [as 别名]
        def getCartoonUrl(self,url):

            if url is None:
                return false
            #todo many decide about url

            try:
                ghost = Ghost()
                #open webkit
                ghost.open(url)
                #exceute javascript and get what you want
                page, resources = ghost.wait_for_page_loaded()
                result, resources = ghost.evaluate("document.getElementById('comic').getAttribute('src');", expect_loading=True)
                del resources
            except Exception,e:
                print e
                return None

# 需要导入模块: from ghost import Ghost [as 别名]
# 或者: from ghost.Ghost import wait_for_page_loaded [as 别名]
#!/usr/bin/python

link = 'http://doodle.com/cqhcwph947kpmfic#table'


from urllib import urlopen
import re


from ghost import Ghost
ghost = Ghost()

page, resources = ghost.open(link)
page, resources = ghost.wait_for_page_loaded()
result, resources = ghost.evaluate(
    "doodleJS.data.poll")

print page
# , resources


# temp = urlopen("http://www.eece.maine.edu/cgi-bin/test-cgi").read()

# pat=re.compile(r'^REMOTE_HOST = (.*)$',re.MULTILINE)
# match=re.search(pat,temp)
# host=match.group(1)

# 需要导入模块: from ghost import Ghost [as 别名]
# 或者: from ghost.Ghost import wait_for_page_loaded [as 别名]
password = ''
placename = ''

if len(sys.argv) != 4:
	print 'Params: username password placename'
	exit(2)
else:
	username = sys.argv[1]
	password = sys.argv[2]
	placename = sys.argv[3]


print "Status: 1"

page, resources = ghost.open('https://foursquare.com/mobile/login?continue=%2Fmobile%2F')
ghost.wait_for_page_loaded()

print "Status: 2"
result, resources = ghost.set_field_value("input[name=username]", username)
result, resources = ghost.set_field_value("input[name=password]", password)
page, resources = ghost.fire_on("form", "submit", expect_loading=True)
ghost.wait_for_page_loaded()


print "Status: 3"
page, resources = ghost.open('https://foursquare.com/mobile/checkin')
ghost.wait_for_page_loaded()


print "Status: 4"
result, resources = ghost.set_field_value("input[name=q]", placename)

# 需要导入模块: from ghost import Ghost [as 别名]
# 或者: from ghost.Ghost import wait_for_page_loaded [as 别名]
def RunExport():
    ghost = Ghost(viewport_size=(1200, 2400), display=False, wait_timeout=30, cache_dir=CACHE_DIRECTORY)#, log_level=logging.ERROR

    #
    #login_password
    #submit.x
    #submit
    page, resources = ghost.open('https://www.paypal.com/ie/cgi-bin/webscr?cmd=_login-run')

    result, resources = ghost.fill("form[name=login_form]", {
            "login_email": PAYPAL_USERNAME,
            "login_password": PAYPAL_PASSWORD
        })
    page, resources = ghost.fire_on("form[name=login_form]", "submit", expect_loading=True)
    result, resources = ghost.wait_for_page_loaded()
    #wait for 10 seconds
    #time.sleep(10)

    page, resources = ghost.open('https://www.paypal.com/ie/cgi-bin/webscr?cmd=_account')


    result, resources = ghost.wait_for_text("Welcome, %s" % PAYPAL_NAME)


    getHistoryListing(ghost)

    first_run = True
    #get the next url
    #print ghost.evaluate('document.querySelectorAll("#tableWrapperID .pagination:nth-child(1) a.btnLink");')[0]
    nav_links_eval = """
                      var links = document.querySelectorAll(".pagination a.btnLink");
                        links.length;
                    """
    nav_links = ghost.evaluate(nav_links_eval)
    page_count = START_AT_PAGE
    transaction_count = 0
    if page_count > 0:
        transaction_count = page_count * 20

    goToPage(ghost,page_count)

    #transaction_list_url = resources[0].url
    #print transaction_list_url
    while nav_links[0] > 0 or first_run==True:
        first_run = False

        page_count = page_count + 1

        filteredlisting_export = os.path.join(EXPORT_DIRECTORY,'filteredhistory%d.png' % page_count)
        if not os.path.isfile(filteredlisting_export):
            ghost.capture_to(filteredlisting_export, selector="body")

        transaction_urls = ghost.evaluate("""
                            var links = document.querySelectorAll("#transactionTable tr.primary td.detailsNoPrint a");
                            var listRet = [];
                            for (var i=0; i<links.length; i++){
                                listRet.push(links[i].href);
                            }
                            listRet;
                            """)


        for transaction_href in transaction_urls[0]:
            transaction_count = transaction_count + 1
            #print urllib.unquote(transaction_href)

            page, resources = ghost.open(urllib.unquote(transaction_href))
            ghost.wait_for_page_loaded()
            payee_name = None
            date_string = None
            date = ghost.evaluate("""
                           document.querySelectorAll("#historyMiniLog tbody tr")[2].querySelectorAll('td')[0].innerHTML;
                        """)
            if date and date[0]:
                date_string = date[0].replace('&nbsp;','')

            payee = ghost.evaluate("""
                           document.querySelectorAll("#historyMiniLog tbody tr")[2].querySelectorAll('td')[1].innerHTML;
                        """)
            if payee and payee[0]:
                payee_name = safeFilename(payee[0].replace('&nbsp;',''))

            if payee_name and date_string:

                date_object = datetime.strptime(date_string, '%d-%b-%Y')
                date_string=datetime.strftime(date_object,'%Y-%m-%d')
                print 'page %d transaction %d [%s - %s]' % (page_count, transaction_count, date_string, payee_name)

                purchasedetails_export = os.path.join(EXPORT_DIRECTORY,'%s_%s_%s.png' % (date_string,payee_name,transaction_count ))

                if not os.path.isfile(purchasedetails_export):
                    print '\t\tsaving to %s' % purchasedetails_export
                    ghost.capture_to(purchasedetails_export, selector="#xptContentMain")
                else:
                    print '\t\tAlready saved to %s' % purchasedetails_export

            else:
                purchasedetails_export = os.path.join(EXPORT_DIRECTORY,'no date and payee - page-%d_ transaction %d.png' % (page_count,transaction_count ))
                print '\t\tsaving to %s' % purchasedetails_export
                if not os.path.isfile(purchasedetails_export):
#.........这里部分代码省略.........