Python Browser.set_handle_refresh方法代码示例

# 需要导入模块: from mechanize import Browser [as 别名]
# 或者: from mechanize.Browser import set_handle_refresh [as 别名]
def init():
    new_form_fac=FormsFactory(form_parser_class=XHTMLCompatibleFormParser)
    new_def_fac=DefaultFactory()
    new_def_fac._forms_factory=new_form_fac
    br=Browser(factory=new_def_fac)

    #br.set_handle_referer(True)
    br.set_handle_refresh(True)
    #br.set_handle_equiv(True)

    ## HTTP Error 403: request disallowed by robots.txt
    br.set_handle_robots(False)
    br.open(login_page)

    ## 'Search' form is form 0
    ## login form is form 1
    br.select_form(nr=1)
    #br.form.set_all_readonly(False)
    #reading from my_secret.txt
    f = open(os.environ['PASSWD_FILE'], 'r')
    passwd=f.readline()
    f.close()
    br['form_loginname']='sitedbrobot'
    br['form_pw']=passwd.replace('\n','')
    br.submit()
    return br

# 需要导入模块: from mechanize import Browser [as 别名]
# 或者: from mechanize.Browser import set_handle_refresh [as 别名]
    def __init__(self, api_key, api_secret, frob, userid, password, perms):
        from mechanize import Browser

        browser = Browser()
        browser.set_handle_robots(False)
        browser.set_handle_refresh(True, max_time=1)
        browser.set_handle_redirect(True)

        api_sig = ''.join((api_secret, 'api_key', api_key, 'frob', frob, 'perms', perms))
        api_sig = md5(api_sig)
        login_url = "http://flickr.com/services/auth/?api_key=%s&perms=%s&frob=%s&api_sig=%s" % (api_key, perms, frob, api_sig)
        browser.open(login_url)
        browser.select_form(name='login_form')
        browser['login'] = userid
        browser['passwd'] = password
        browser.submit()
        for form in browser.forms():
            try:
                if form['frob'] == frob:
                    browser.form = form
                    browser.submit()
                    break
            except:
                pass
        else:
            raise Exception('no form for authentication found')  # lame :-/

# 需要导入模块: from mechanize import Browser [as 别名]
# 或者: from mechanize.Browser import set_handle_refresh [as 别名]
def new_browser():
    b = Browser()

    b.set_handle_robots(False)
    b.set_handle_refresh(False)
    b.addheaders = [('User-agent', 'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.1) Gecko/2008071615 Fedora/3.0.1-1.fc9 Firefox/3.0.1')]
    
    return b

# 需要导入模块: from mechanize import Browser [as 别名]
# 或者: from mechanize.Browser import set_handle_refresh [as 别名]
def init_browser():
    browser = Browser()
    browser.addheaders = (
        ('Accept', 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'),
        ('User-agent', 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0)') # We're Firefox! :P
    )
    # browser.set_handle_gzip(True) # Currently experimental in mechanize
    browser.set_handle_redirect(True)
    browser.set_handle_refresh(False)
    browser.set_handle_robots(True)
    browser.set_handled_schemes(['http', 'https'])
    browser.set_proxies({})
    return browser

# 需要导入模块: from mechanize import Browser [as 别名]
# 或者: from mechanize.Browser import set_handle_refresh [as 别名]
def lockNloadBrowser():
    br = Browser()
    cj = cookielib.LWPCookieJar()
    br.set_cookiejar(cj)
    br.set_handle_equiv(True)
    br.set_handle_gzip(True)
    br.set_handle_redirect(True)
    br.set_handle_referer(True)
    br.set_handle_robots(False)

    # Follows refresh 0 but not hangs on refresh > 0
    br.set_handle_refresh(mechanize._http.HTTPRefreshProcessor(), max_time=1)
    br.addheaders = [('User-agent', 'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.1) Gecko/2008071615 Fedora/3.0.1-1.fc9 Firefox/3.0.1')]
    return br

# 需要导入模块: from mechanize import Browser [as 别名]
# 或者: from mechanize.Browser import set_handle_refresh [as 别名]
class Api:
    appid = '5415093'
    token = None
    query_pattern = 'https://api.vk.com/method/%s?%s&access_token='

    ua = 'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.1) Gecko/2008071615 Fedora/3.0.1-1.fc9 Firefox/3.0.1'
    br = None
    cl = None

    def __init__(self, login, password, scope, testmode=False):
        self.br = Browser()
        self.cl = cookielib.LWPCookieJar()
        self.br.set_cookiejar(self.cl)

        self.br.set_handle_equiv(True)
        self.br.set_handle_redirect(True)
        self.br.set_handle_referer(True)
        self.br.set_handle_robots(False)
        self.br.set_handle_refresh(_http.HTTPRefreshProcessor(), max_time=1)
        self.br.addheaders = [('User-agent', self.ua)]

        self.br.open('https://oauth.vk.com/authorize?client_id=' + self.appid +
                     '&scope=' + scope + '&redirect_uri=http://oauth.vk.com/blank.html' +
                     '&display=mobile&response_type=token')

        self.br.select_form(nr=0)
        self.br.form['email'] = login
        self.br.form['pass'] = password
        self.br.submit()

        if len(list(self.br.forms())) > 0:
            self.br.select_form(nr=0)
            self.br.submit()

        params = urlparse.urlparse(self.br.geturl()).fragment
        params = params.split('&')

        for val in params:
            tp = val.split('=')
            if tp[0] == 'access_token':
                self.token = tp[1]
                self.query_pattern += self.token
                if testmode:
                    self.query_pattern += '&test_mode=1'
                break

    def query(self, func, data):
        response = self.br.open(self.query_pattern % (func, data))
        return response.read()

# 需要导入模块: from mechanize import Browser [as 别名]
# 或者: from mechanize.Browser import set_handle_refresh [as 别名]
def doLogin():
    global br 
    br = Browser()
    br.set_handle_robots(False)
    br.set_handle_refresh(False)
    br.addheaders = [('User-agent', 'Firefox')]
    br.open("https://www.qruiser.com")
    #response1 = br.follow_link(link)
    i = 0
    for form in br.forms():
        i += 1
        #print form
        if i > 1:
            br.form = form
    br.form['loginname'] = 'maxberggren'
    br.form['loginpassword'] = 'wig6?rEPA'
    response = br.submit()  

# 需要导入模块: from mechanize import Browser [as 别名]
# 或者: from mechanize.Browser import set_handle_refresh [as 别名]
def main():
    br = Browser()
    br.set_handle_robots(False)
    br.set_handle_refresh(False)
    br.addheaders = config.USER_AGENT

    d = shelve.open("results.cache")

    for school in config.SCHOOLS:
        points, courses = crawl(br, config.BASE_URL + school, school)
        if not points:
            continue
        if config.VERBOSE:
            printstdout(school, points, courses)

        # We have data stored from before
        # We will check if for new courses
        # Data format: shelve: school = {points: num, courses: []}
        if school in d:
            try:
                c = new_courses(courses, d[school]['courses'])
            except:
                c = None
            if c:
                # Update stored courses and points
                d[school] = {}  # Cleaning to be able to update
                d[school]['courses'] = courses
                d[school]['points'] = points

                # Notify if new courses and notifications are turned on
                if config.NOTIFY:
                    notify(c)
            else:
                # No data from before. Storing it
                data = {}
                data['points'] = points
                data['courses'] = courses
                d[school] = data
        else:
            # No data from before. Storing it
            data = {}
            data['points'] = points
            data['courses'] = courses
            d[school] = data
    d.close()

# 需要导入模块: from mechanize import Browser [as 别名]
# 或者: from mechanize.Browser import set_handle_refresh [as 别名]
 def login(self):
     br = Browser()
     cj = cookielib.LWPCookieJar()
     br.set_cookiejar(cj)
     
     br.set_handle_equiv(True)
     br.set_handle_redirect(True)
     br.set_handle_referer(True)
     br.set_handle_robots(False)
     br.set_handle_refresh(_http.HTTPRefreshProcessor(), max_time=2)
     
     br.open('http://www.fatsecret.com/Auth.aspx?pa=s')
     br.select_form(nr=0)
     #name attr of login tr
     #PLEASE input your username and password here!!!!
     br['_ctl0:_ctl7:Logincontrol1:Name'] = 'username'
     br['_ctl0:_ctl7:Logincontrol1:Password'] = 'password'
     br.submit()
     return br

# 需要导入模块: from mechanize import Browser [as 别名]
# 或者: from mechanize.Browser import set_handle_refresh [as 别名]
def get_br():
    #todo low
    #headers
    #Accept-Encoding: identity
    # Host: _login.weibo.cn
    # Referer: http://weibo.cn/
    # Connection: close
    # User-Agent: Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0)
    br = Browser(factory=RobustFactory(), history=NoHistory(),)
    cj = cookielib.LWPCookieJar()
    br.back = back_func
    br.set_cookiejar(cj)
    br.set_handle_equiv(True)
    #br.set_handle_gzip(True) #gzip在mechanize里面还不是正式功能
    br.set_handle_redirect(True)
    br.set_handle_referer(True)
    br.set_handle_robots(False)
    br.set_handle_refresh(HTTPRefreshProcessor(), max_time=10)
    br.addheaders = [('User-agent', USER_AGENT)]
    return br

# 需要导入模块: from mechanize import Browser [as 别名]
# 或者: from mechanize.Browser import set_handle_refresh [as 别名]
 def __init__(self,browser,cookies):
     from mechanize import Browser
     br = Browser()
     # Cookie Jar
     if cookies == "firefox": cj = getFirefoxCookie()
     else: cj=set_cookies_from_text(cookies)
     br.set_cookiejar(cj)
     # Browser options 
     br.set_handle_equiv(True) 
     #br.set_handle_gzip(True) 
     br.set_handle_redirect(True) 
     br.set_handle_referer(True) 
     br.set_handle_robots(False)
     # Debug Options
     #br.set_debug_http(True) 
     #br.set_debug_redirects(True) 
     #br.set_debug_responses(True)
     # Follows refresh 0 but not hangs on refresh > 0 
     br.set_handle_refresh(mechanize._http.HTTPRefreshProcessor(), max_time=1)
     # User-Agent
     br.addheaders = [('user-agent', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.45 Safari/537.36'),
     ('accept', 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8')]
     self.browser = br
     self.browserType = 'mechanize'

# 需要导入模块: from mechanize import Browser [as 别名]
# 或者: from mechanize.Browser import set_handle_refresh [as 别名]
from mechanize import Browser
import pprint
import sys, re, os

from SiteDB import getSiteDBSiteNames

login_page='https://savannah.cern.ch/account/login.php?uri=%2F'
create_site='https://savannah.cern.ch/support/admin/field_values.php?group=cmscompinfrasup&list_value=1&field=custom_sb1'
#site_list='https://cmsweb.cern.ch/sitedb/reports/showReport?reportid=naming_convention.ini'
new_form_fac=FormsFactory(form_parser_class=XHTMLCompatibleFormParser)
new_def_fac=DefaultFactory()
new_def_fac._forms_factory=new_form_fac
br=Browser(factory=new_def_fac)

#br.set_handle_referer(True)
br.set_handle_refresh(True)
#br.set_handle_equiv(True)

## HTTP Error 403: request disallowed by robots.txt
br.set_handle_robots(False)

#lets get a list of CMS names from SiteDB
label="Getting site names from SiteDB".ljust(50,'.')
sys.stdout.write(label[0:40])
sys.stdout.flush()
siteNamesFromSiteDB = getSiteDBSiteNames()

siteNamesFromSiteDB=[x[1] for x in siteNamesFromSiteDB]
for site in siteNamesFromSiteDB:
  if site.endswith('_Disk'):
    siteNamesFromSiteDB.remove(site)

# 需要导入模块: from mechanize import Browser [as 别名]
# 或者: from mechanize.Browser import set_handle_refresh [as 别名]
class LconnectScraper(ClassDataScraper):
    LCONNECT_URL = 'http://leopardweb.wit.edu/'
    USERAGENT = 'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.2.1) ' \
                + 'Gecko/20100122 firefox/3.6.1'

    def __init__(self):
        # Create a cookie jar and a browser
        self._cookieJar = LWPCookieJar()
        self._browser = Browser()
        self._browser.set_cookiejar(self._cookieJar)

        # Set Browser options
        self._browser.set_handle_equiv(True)
        self._browser.set_handle_gzip(True)
        self._browser.set_handle_redirect(True)
        self._browser.set_handle_referer(True)
        self._browser.set_handle_robots(False)
        self._browser.set_handle_refresh(_http.HTTPRefreshProcessor(),
                                         max_time=1)
        self._browser.addheaders = [('User-agent', LconnectScraper.USERAGENT)]

        # Debugging
        self._browser.set_debug_http(True)
        self._browser.set_debug_redirects(True)
        self._browser.set_debug_responses(True)

    def getName(self):
        return "Lconnect Scraper"

    def connect(self):
        """
        Attempts to connect to the data source
        """
        try:
            # Try to open a connection. 8 Second timeout
            self._browser.open(LconnectScraper.LCONNECT_URL, timeout=8)
            return True
        except URLError:
            return False

    def disconnect(self):
        """
        Disconnects from the data source
        """

        self._browser.close()

    def requiresAuthentication(self):
        """
        Returns whether or not the scraper requires authentication information
        """

        return True

    def authenticate(self, username, password):
        """
        Attempts to authenticate the scraper using username and password
        """

        # If we're on the sign in page, try to sign in
        if self._browser.title() == 'Sign In':
            for form in self._browser.forms():
                if form.name is None:
                    self._browser.form = list(self._browser.forms())[0]
                    self._browser['username'] = username
                    self._browser['password'] = password

                    self._browser.submit()

        # If the browser's title is 'Main Menu',
        # we've either successfully logged in, or we were already
        if self._browser.title() == 'Main Menu':
            return True
        else:
            return False

    def getClassData(self):
        """
        Returns a list of ClassData objects
        """

        return []

# 需要导入模块: from mechanize import Browser [as 别名]
# 或者: from mechanize.Browser import set_handle_refresh [as 别名]
from BeautifulSoup import BeautifulSoup
import cookielib
import json

#### Start Config ##################
browser_client = Browser()
web_locations = "http://facebook.com/" #Main Web Url
login_page = "login.php"  #Path to login page in main web
user_page = "home.php"   #Path to home page in main web
cj = cookielib.LWPCookieJar()
browser_client.set_cookiejar(cj)
browser_client.set_handle_equiv(True)
browser_client.set_handle_redirect(True)
browser_client.set_handle_referer(True)
browser_client.set_handle_robots(False)
browser_client.set_handle_refresh(_http.HTTPRefreshProcessor(), max_time=1)
browser_client.addheaders = [('User-agent', 'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.2.6) Gecko/20100720 Gentoo Firefox/3.6.6')]
#### End Config ####################

def facebook_login(input_username , input_password):
    user = input_username
    password = input_password
    page = web_locations+login_page
    browser_client.open(page)
    browser_client.select_form(nr=0)
    browser_client["email"]=user
    browser_client["pass"]=password
    browser_client.submit()
    next_page = web_locations+user_page
    browser_client.open(next_page)

# 需要导入模块: from mechanize import Browser [as 别名]
# 或者: from mechanize.Browser import set_handle_refresh [as 别名]
def login():

    creds = get_credentials()
    if not creds:
        return None

    b = Browser()
    b.set_handle_robots(False)
    b.addheaders = [
        ('User-agent', 'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.1) Gecko/2008071615 Fedora/3.0.1-1.fc9 Firefox/3.0.1'),
        ('Connection', 'keep-alive'),
        ('Cache-Control', 'max-age=0'),
        ('Accept-Encoding', 'gzip, deflate, br')
    ]

    b.set_handle_equiv(True)
    b.set_handle_gzip(True)
    b.set_handle_redirect(True)
    b.set_handle_referer(True)
    b.set_handle_robots(False)

    # Follows refresh 0 but not hangs on refresh > 0
    b.set_handle_refresh(_http.HTTPRefreshProcessor(), max_time=1)

    # Want debugging messages?
    # b.set_debug_http(True)
    b.set_debug_redirects(True)
    b.set_debug_responses(True)
    print 'Opening main page...'
    b.open('http://www.nab.com.au')
    print 'OK'

    print 'Opening login redir page...'
    b.open('http://www.nab.com.au/cgi-bin/ib/301_start.pl?browser=correct')
    print 'OK'

    print 'Opening real login page...'
    b.open('https://ib.nab.com.au/nabib/index.jsp')
    print 'OK'

    b.select_form(nr=0)
    try:
        webKeyCtrl = b.form.find_control(id='webKey')
        webAlphaCtrl = b.form.find_control(id='webAlpha')
    except ControlNotFoundError:
        print 'Cannot find necessary login controls, quitting'
        return

    webKey = webKeyCtrl.value
    webAlpha = webAlphaCtrl.value
    newPassword = make_password(creds[1], webKey, webAlpha)

    usernameCtrl = b.form.find_control(name='userid')
    passwordCtrl = b.form.find_control(name='encoded-password')
    passwordCtrl.readonly = False
    usernameCtrl.value = creds[0]
    passwordCtrl.value = newPassword

    rawPassword = b.form.find_control(name='password')
    rawPassword.value = ''

    b_data = b.form.find_control(name='browserData')
    b_data.readonly = False
    b_data.value = '1496488636702;z=-600*-600;s=1440x900x24;l=en-GB;p=MacIntel;h=1Z3uS;i=33;j=117;k=16;c=d3d3Lm5hYi5jb20uYXUvc3RhdGljL0lCL2xvZ2luQmFubmVyLw;n=bG9naW5Gb3Jt,bG9naW5UaXBz;e=Y3ZpZXcz;b=1JE4yQ,24uNEg,2wDBVE;a=1GeUEa,1TaPsP,1ZO-16,1rEqxh,2.jbKy,21b2P5,2Jrfu6,2LmSef,2TqVCf,2Ubrnm,2dgqqB,3MkcJZ,JIGdn,eqyBa,lTM8m;o=Y29uc29sZQ,Y2hyb21l,YW5ndWxhcg,YXBpTG9nb3V0QXBw,Z2V0QnJvd3Nlcg,alF1ZXJ5MTEwMjA4MzYwNzIxMDQ4NTY0MjY0;t=fo4f0ot8-600.j3h6ekzf.877;d=YWNz,Ym9keWNvbnRhaW5lcg,Ym9keWNvbnRhaW5lcl9pbnNpZGU,YmFubmVy,ZXJyb3JNZXNzYWdl,ZXJyb3JOdW1iZXI,Zm9vdGVyX2xvZ2lu,ZmFuY3ktYmctZQ,ZmFuY3ktYmctbg,ZmFuY3ktYmctbmU,ZmFuY3ktYmctbnc,ZmFuY3ktYmctc2U,ZmFuY3ktYmctc3c,ZmFuY3ktYmctcw,ZmFuY3ktYmctdw,ZmFuY3lib3gtY2xvc2U,ZmFuY3lib3gtaW5uZXI,ZmFuY3lib3gtb3V0ZXI,ZmFuY3lib3gtb3ZlcmxheQ,ZmFuY3lib3gtbG9hZGluZw,ZmFuY3lib3gtbGVmdA,ZmFuY3lib3gtbGVmdC1pY28,ZmFuY3lib3gtcmlnaHQ,ZmFuY3lib3gtcmlnaHQtaWNv,ZmFuY3lib3gtd3JhcA,ZmFuY3lib3gtdG1w,aGVhZGVy,aWItdXNlci10ZXh0,bG9naW5Gb3Jt,bGlua3Mtc29jaWFsLW1lZGlh,bWFpblBhZ2U;u=ZHVtbXk,ZW5jb2RlZC1wYXNzd29yZA,d2ViQWxwaGE,d2ViS2V5;v=bmVlZC1oZWxw;x=1IVClf,1KxWAP,1SURBl,1Wl6jj,1vhE2s,1vstXM,1wlzQT,1yYwT1,2-PmTs,2APt-x,2FOxw2,2Lnxl,2ceYJE,2feZ0x,2g4LgQ,2h079f,2oK-0A,2ueFc7,34liSK,39CTWT,3GxyfT,3T6P3H,3XvqP.,3kcnCG,3ktPLw,3l39dK,660SR,68npD,8Vcav,JOS8B,cTezC,dwOmq,ix9Ek,s-ZAp;q=ZnJhdWQ;w=428866'

    b.form.new_control('text', 'login', {'value': ''})
    b.form.fixup()
    b['login'] = 'Login'

    print('Logging in...')
    b.submit()

    if not check_url(b, logged_in_urls):
        print('Error logging in.')
        return None

    print('OK')
    return b