Python Browser.set_handle_equiv方法代码示例

# 需要导入模块: from mechanize import Browser [as 别名]
# 或者: from mechanize.Browser import set_handle_equiv [as 别名]
def find_first_article():
    mech = Browser()
    cj = cookielib.LWPCookieJar()

    mech.set_handle_equiv(True)
    # mech.set_handle_gzip(True)
    mech.set_handle_redirect(True)
    mech.set_handle_referer(True)
    mech.set_handle_robots(False)
    # mech.set_handle_refresh(mechanize._http.HTTPRefreshProcessor(), max_time=1)
    mech.addheaders = [
        (
            "User-agent",
            "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.1) Gecko/2008071615 Fedora/3.0.1-1.fc9 Firefox/3.0.1",
        )
    ]

    page = mech.open("https://bitcointalk.org/index.php?board=77.0")
    html = page.read()

    soup = BeautifulSoup(html)

    first_article_tag = soup.find("td", class_="windowbg")

    global startingpost
    startingpost = first_article_tag.span.a.get("href")
    print startingpost

# 需要导入模块: from mechanize import Browser [as 别名]
# 或者: from mechanize.Browser import set_handle_equiv [as 别名]
def returnMnemonics(var):
    from mechanize import Browser
    from bs4 import BeautifulSoup
    # var = "abase"
    br = Browser()
    br.set_handle_robots(False)
    br.set_handle_equiv(False)
    br.addheaders = [('User-agent', 'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.1) Gecko/2008071615 Fedora/3.0.1-1.fc9 Firefox/3.0.1')]
    url= 'http://mnemonicdictionary.com/?word=' + str(var)
    br.open(url)

    soup_mn = BeautifulSoup(br.response().read())

    # <div style="padding-top: 10px;">
    count_mn=0
    mnemonics=""
    for i in soup_mn.find_all('div',{'style':'padding-top: 10px;'}):

        soup2 = BeautifulSoup(str(i))
        for x in soup2.find_all('div', {'class':'row-fluid'}):
            soup3 = BeautifulSoup(str(x))

            for y in soup3.find_all('div', {'class':'span9'}):
                count = 0
                # print count_mn
                if count_mn==3:
                    break
                count_mn = count_mn+1
                if y is not None:
                    for z in y:
                        if count == 2:
                            # print z
                            mnemonics = mnemonics+z.strip().replace(','," ").replace('\n', '').replace(".","")+","
                        count = count+1
    return mnemonics

# 需要导入模块: from mechanize import Browser [as 别名]
# 或者: from mechanize.Browser import set_handle_equiv [as 别名]
def get_machines(start,num_pages):
    mech = Browser()
    mech.set_handle_robots(False)
    mech.set_handle_equiv(False) 
    mech.addheaders = [('User-agent', 'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.1) Gecko/2008071615 Fedora/3.0.1-1.fc9 Firefox/3.0.1')]

    machines = []
    try:
        page_num = 0
        for page_num in range(start,num_pages+1):
            print("page %d" % (page_num))
            url = "http://www.pinpedia.com/machine?page=%d" % page_num
            html_page = mech.open(url)
            html = html_page.read()
            machines += parse_page(html)
            time.sleep(0.1)
    except Exception as e:
        print e
        print("finished at page %s" % page_num)

    print("storing machines to machines.txt")

    with open('machines.txt','w') as fh:
        for machine in machines:
            fh.write(machine + "\n")

# 需要导入模块: from mechanize import Browser [as 别名]
# 或者: from mechanize.Browser import set_handle_equiv [as 别名]
def extract_article_url(posturl):
    mech = Browser()
    cj = cookielib.LWPCookieJar()

    mech.set_handle_equiv(True)
    # mech.set_handle_gzip(True)
    mech.set_handle_redirect(True)
    mech.set_handle_referer(True)
    mech.set_handle_robots(False)
    # mech.set_handle_refresh(mechanize._http.HTTPRefreshProcessor(), max_time=1)
    mech.addheaders = [
        (
            "User-agent",
            "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.1) Gecko/2008071615 Fedora/3.0.1-1.fc9 Firefox/3.0.1",
        )
    ]

    page = mech.open(posturl)
    html = page.read()

    global soup
    soup = BeautifulSoup(html)

    global articleURL
    # print soup.prettify()

    for item in soup.find_all("div", class_="post"):
        for link in item.find_all("a"):
            string = link.get("href")
            if prog.match(string):
                # find the link that is to the article (link outside of bitcointalk.org forum)
                articleURL = link.get("href")
                return link.get("href")
    return "No article url"

# 需要导入模块: from mechanize import Browser [as 别名]
# 或者: from mechanize.Browser import set_handle_equiv [as 别名]
    def get_browser(self):
        """
        Each FAUrl object stores it's own browser instance. On the first call
         it is created and if the username and password is set it will
         authenticate you.

        :return: mechanize.Browser instance.
        :raise: FAiler.FAError if FA is down. Time to F5!
        :raise: FAiler.FAAuth Your username and password failed
        """
        if self._br is None:
            br = Browser()
            br.set_handle_robots(False)
            br.set_handle_redirect(True)
            br.set_handle_referer(True)
            br.set_handle_equiv(True)
            br.addheaders = [('User-agent', 'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.1) Gecko/2008071615 Fedora/3.0.1-1.fc9 Firefox/3.0.1')]
            if self._username is not None and self._password is not None:
                loginPage = 'https://www.furaffinity.net/login'
                try:
                    br.open(loginPage)
                except urllib2.HTTPError:
                    raise FAError("FA's down, F5 time.")
                br.form = br.global_form()
                br.form['name'] = self._username
                br.form['pass'] = self._password
                br.form.method = 'POST'
                br.submit()
                if br.geturl() == loginPage + '/?msg=1':
                    raise FAAuth('Username & Password Incorrect')
            self._br = br
        return self._br

# 需要导入模块: from mechanize import Browser [as 别名]
# 或者: from mechanize.Browser import set_handle_equiv [as 别名]
def get_browser():
    # Browser
    br = Browser()

    # Cookie Jar
    #cj = cookielib.LWPCookieJar()
    #br.set_cookiejar(cj)

    # Browser options
    br.set_handle_equiv(True)
    br.set_handle_gzip(True)
    br.set_handle_redirect(True)
    br.set_handle_referer(True)
    br.set_handle_robots(False)

    # Follows refresh 0 but not hangs on refresh > 0
    #br.set_handle_refresh(mechanize._http.HTTPRefreshProcessor(), max_time=1)

    # Want debugging messages?
    #
    #br.set_debug_http(True)
    #br.set_debug_redirects(True)
    #br.set_debug_responses(True)

    # User-Agent (this is cheating, ok?)
    br.addheaders = [('User-agent', 'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.1) Gecko/2008071615 Fedora/3.0.1-1.fc9 Firefox/3.0.1')]

    return br

# 需要导入模块: from mechanize import Browser [as 别名]
# 或者: from mechanize.Browser import set_handle_equiv [as 别名]
def generateSentence(var):
    br = Browser()
    br.set_handle_robots(False)
    br.set_handle_equiv(False)
    br.addheaders = [('User-agent', 'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.1) Gecko/2008071615 Fedora/3.0.1-1.fc9 Firefox/3.0.1')]
    url= 'http://www.oxforddictionaries.com/definition/english/' + str(var)
    # url= 'https://www.google.co.in/search?q=define+utilitarian'
    try:
        br.open(url)
    except:
        print "what word is this, man? " + var
        return
    soup = BeautifulSoup(br.response().read())
    sentence=""
    counter=0
    for i in soup.find_all('ul',{'class':'sentence_dictionary'}):
        if i is not None:
            soup2 = BeautifulSoup(str(i))
            for j in soup2.find_all('li',{'class':'sentence'}):
                if j is not None:
                    sentence = sentence + str(counter+1)+") "+j.string.replace(',',' ').strip()+"\n"
                    counter+=1
                    if counter == 2:
                        return sentence
    return sentence

# 需要导入模块: from mechanize import Browser [as 别名]
# 或者: from mechanize.Browser import set_handle_equiv [as 别名]
class GetPIN():
    def __init__(self,url,username, password):
        self.br = Browser()
        self.br.set_handle_equiv(False)
        self.br.set_handle_robots(False)
        self.url = url
        self.username = username
        self.password = password
 
    def getPIN(self):
        self.br.open(self.url)
        try:
            self.br.select_form(name="authZForm")
            self.br['userId'] = self.username
            self.br['passwd'] = self.password
            response = self.br.submit()
            data = response.readlines()
        except:
            data = self.br.response().readlines()
        pattern = r'<span class="fb">(.*?)</span>' 
        pat = re.compile(pattern)
        for line in data:
            if pat.search(line):
                verifier = pat.findall(line)
                break
        if len(verifier):
            return verifier[0]
        else:
            return -1

# 需要导入模块: from mechanize import Browser [as 别名]
# 或者: from mechanize.Browser import set_handle_equiv [as 别名]
def browserInit():
    '''
    Set standard, initial browser configuration.
    '''
    browser = Browser()
    browser.set_handle_equiv(True)
    browser.set_handle_redirect(True)
    browser.set_handle_referer(True)
    browser.set_handle_robots(False)
    browser.addheaders = [('user-agent', '   Mozilla/5.0 (X11; U; Linux x86_64; en-US) Mechanize/0.2.4 Fedora/16 (Verne) Pytane/0.2'),
('accept', 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8')]
    return browser

# 需要导入模块: from mechanize import Browser [as 别名]
# 或者: from mechanize.Browser import set_handle_equiv [as 别名]
def lockNloadBrowser():
    br = Browser()
    cj = cookielib.LWPCookieJar()
    br.set_cookiejar(cj)
    br.set_handle_equiv(True)
    br.set_handle_gzip(True)
    br.set_handle_redirect(True)
    br.set_handle_referer(True)
    br.set_handle_robots(False)

    # Follows refresh 0 but not hangs on refresh > 0
    br.set_handle_refresh(mechanize._http.HTTPRefreshProcessor(), max_time=1)
    br.addheaders = [('User-agent', 'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.1) Gecko/2008071615 Fedora/3.0.1-1.fc9 Firefox/3.0.1')]
    return br

# 需要导入模块: from mechanize import Browser [as 别名]
# 或者: from mechanize.Browser import set_handle_equiv [as 别名]
class Api:
    appid = '5415093'
    token = None
    query_pattern = 'https://api.vk.com/method/%s?%s&access_token='

    ua = 'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.1) Gecko/2008071615 Fedora/3.0.1-1.fc9 Firefox/3.0.1'
    br = None
    cl = None

    def __init__(self, login, password, scope, testmode=False):
        self.br = Browser()
        self.cl = cookielib.LWPCookieJar()
        self.br.set_cookiejar(self.cl)

        self.br.set_handle_equiv(True)
        self.br.set_handle_redirect(True)
        self.br.set_handle_referer(True)
        self.br.set_handle_robots(False)
        self.br.set_handle_refresh(_http.HTTPRefreshProcessor(), max_time=1)
        self.br.addheaders = [('User-agent', self.ua)]

        self.br.open('https://oauth.vk.com/authorize?client_id=' + self.appid +
                     '&scope=' + scope + '&redirect_uri=http://oauth.vk.com/blank.html' +
                     '&display=mobile&response_type=token')

        self.br.select_form(nr=0)
        self.br.form['email'] = login
        self.br.form['pass'] = password
        self.br.submit()

        if len(list(self.br.forms())) > 0:
            self.br.select_form(nr=0)
            self.br.submit()

        params = urlparse.urlparse(self.br.geturl()).fragment
        params = params.split('&')

        for val in params:
            tp = val.split('=')
            if tp[0] == 'access_token':
                self.token = tp[1]
                self.query_pattern += self.token
                if testmode:
                    self.query_pattern += '&test_mode=1'
                break

    def query(self, func, data):
        response = self.br.open(self.query_pattern % (func, data))
        return response.read()

# 需要导入模块: from mechanize import Browser [as 别名]
# 或者: from mechanize.Browser import set_handle_equiv [as 别名]
def getOneExperience(q):
    while True:

        url = q.get()
        print url
        pathToFile = "C:\\Users\\amukhopadhyay\\Desktop\\gfg.html"
        writeToFile=""

        br = Browser()
        br.set_handle_robots(False)
        br.set_handle_equiv(False)
        br.addheaders = [('User-agent', 'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.1) Gecko/2008071615 Fedora/3.0.1-1.fc9 Firefox/3.0.1')]
        # url= raw_input("givepath:")
        # url = "http://www.geeksforgeeks.org/amazon-interview-experience-set-158-off-campus/"
        br.open(url)

        soup = BeautifulSoup(str(br.response().read()), from_encoding="utf-8")
        # print(soup.prettify(formatter=substitute_html_entities))
        soup = BeautifulSoup(soup.prettify(formatter=substitute_html_entities))

        for i in soup.find_all('h2',{'class':'post-title'}, limit=1):
            writeToFile +="<h2>"+str(i.string)+"</h2>"

        flag=0
        for i in soup.find_all('div',{'id':'content'}):
            # print i.prettify()
            soup2 = BeautifulSoup(i.prettify())
            if flag == 1:
                break
            for j in soup2.find_all('p'):
                if flag == 1:
                    break
                if j.string is not None:
                    if len(j.string.strip()) >25:
                        if j.string.strip()[:25] == "If you like GeeksforGeeks":
                            flag=1
                            j="\n\n\n"
                writeToFile+=(j.encode('utf-8').replace("\'",""))


        MY_LOCK.acquire()
        with open(pathToFile, 'a') as f:
            f.write(writeToFile)
            f.close()
        MY_LOCK.release()
        print url, "done"
        q.task_done()
        return

# 需要导入模块: from mechanize import Browser [as 别名]
# 或者: from mechanize.Browser import set_handle_equiv [as 别名]
def create_browser(debug=False):
    browser = Browser(factory=mechanize.RobustFactory())
    if debug:
        # Maybe enable this if you want even more spam...
        # logger = logging.getLogger("mechanize")
        # logger.addHandler(logging.StreamHandler(sys.stdout))
        # logger.setLevel(logging.DEBUG)
        browser.set_debug_http(True)
        browser.set_debug_responses(True)
        browser.set_debug_redirects(True)
    browser.set_handle_equiv(True)
    browser.set_handle_gzip(True)
    browser.set_handle_redirect(True)
    browser.set_handle_referer(True)
    browser.set_handle_robots(False)
    browser.addheaders = HEADERS
    return browser

# 需要导入模块: from mechanize import Browser [as 别名]
# 或者: from mechanize.Browser import set_handle_equiv [as 别名]
def fetch(name):
    mech = Browser()
    mech.set_handle_robots(False)
    mech.set_handle_equiv(False) 
    mech.addheaders = [('User-agent', 'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.1) Gecko/2008071615 Fedora/3.0.1-1.fc9 Firefox/3.0.1')]

    try:
        page_num = 0
        for page_num in range(100):
            print("page %d for %s" % (page_num,name))
            url = "http://www.pinpedia.com/machine/%s/prices?page=%d" % (name,page_num)
            print url
            html_page = mech.open(url)
            html = html_page.read()
            with open('%s/%s-%s.html' % (html_dir,name,page_num),'w') as fh:
                fh.write(html)
            time.sleep(0.5)
    except Exception:
        print("finished at page %s" % page_num)

# 需要导入模块: from mechanize import Browser [as 别名]
# 或者: from mechanize.Browser import set_handle_equiv [as 别名]
 def login(self):
     br = Browser()
     cj = cookielib.LWPCookieJar()
     br.set_cookiejar(cj)
     
     br.set_handle_equiv(True)
     br.set_handle_redirect(True)
     br.set_handle_referer(True)
     br.set_handle_robots(False)
     br.set_handle_refresh(_http.HTTPRefreshProcessor(), max_time=2)
     
     br.open('http://www.fatsecret.com/Auth.aspx?pa=s')
     br.select_form(nr=0)
     #name attr of login tr
     #PLEASE input your username and password here!!!!
     br['_ctl0:_ctl7:Logincontrol1:Name'] = 'username'
     br['_ctl0:_ctl7:Logincontrol1:Password'] = 'password'
     br.submit()
     return br