Python mechanicalsoup.StatefulBrowser方法代码示例

# 需要导入模块: import mechanicalsoup [as 别名]
# 或者: from mechanicalsoup import StatefulBrowser [as 别名]
def get_current_usd_to_cny():
    """
    Get the current China mainland bank transfer buying rate for USD to CNY.

    Casting returned objects to string can ensure they do not inadvertently contain
    a BS4 object, which presents difficulties in pickling which can be hard to debug.
    If you do use newt.db, scrape objects can be tested with newt.db.jsonpickle
    `dumps` function, to ensure that the object can be both pickled and also
    serialized to json by newt.db and indexed/saved in a PostgreSQL jsonb field.

    :return: str(): rate in CNY, str(): time string
    """
    browser = StatefulBrowser()
    browser.open('http://www.boc.cn/sourcedb/whpj/enindex.html')
    trs = browser.get_current_page().find_all("tr")
    cells = _get_usd_row_cells(trs)
    rate = cells[0].text
    time = cells[5].text
    time = time.split()
    time = time[0] + ' ' + time[1]
    return str(rate), str(time) 

# 需要导入模块: import mechanicalsoup [as 别名]
# 或者: from mechanicalsoup import StatefulBrowser [as 别名]
def __init__(self):

        # Whether we already have a valid HTTP session with the remote server
        self.http_session_valid = False

        # PEP 476: verify HTTPS certificates by default (implemented from Python 2.7.9)
        # https://bugs.python.org/issue22417
        if sys.hexversion >= 0x02070900:
            import ssl
            ssl._create_default_https_context = ssl._create_unverified_context

        # https://mechanicalsoup.readthedocs.io/
        self.browser = mechanicalsoup.StatefulBrowser()

        # Set custom user agent header
        self.browser.set_user_agent('Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.112 Safari/537.36')

        self.searchurl = self.baseurl + 'einsteiger' #?lang={language}'
        self.accessurl = self.baseurl + 'register:showalleverfahrenstabellen?AKZ={number}&lang={language}'

        self.reference_pattern = re.compile('.*\?AKZ=(.+?)&.*') 

# 需要导入模块: import mechanicalsoup [as 别名]
# 或者: from mechanicalsoup import StatefulBrowser [as 别名]
def test_form_not_found():
    browser = mechanicalsoup.StatefulBrowser()
    browser.open_fake_page(page_with_various_fields)
    form = browser.select_form('form')
    with pytest.raises(mechanicalsoup.utils.LinkNotFoundError):
        form.input({'foo': 'bar', 'nosuchname': 'nosuchval'})
    with pytest.raises(mechanicalsoup.utils.LinkNotFoundError):
        form.check({'foo': 'bar', 'nosuchname': 'nosuchval'})
    with pytest.raises(mechanicalsoup.utils.LinkNotFoundError):
        form.check({'entree': 'cheese'})
    with pytest.raises(mechanicalsoup.utils.LinkNotFoundError):
        form.check({'topping': 'tofu'})
    with pytest.raises(mechanicalsoup.utils.LinkNotFoundError):
        form.textarea({'bar': 'value', 'foo': 'nosuchval'})
    with pytest.raises(mechanicalsoup.utils.LinkNotFoundError):
        form.set_radio({'size': 'tiny'})
    with pytest.raises(mechanicalsoup.utils.LinkNotFoundError):
        form.set_select({'entree': ('no_multiple', 'no_multiple')}) 

# 需要导入模块: import mechanicalsoup [as 别名]
# 或者: from mechanicalsoup import StatefulBrowser [as 别名]
def test_links():
    browser = mechanicalsoup.StatefulBrowser()
    html = '''<a class="bluelink" href="/blue" id="blue_link">A Blue Link</a>
              <a class="redlink" href="/red" id="red_link">A Red Link</a>'''
    expected = [BeautifulSoup(html, "lxml").a]
    browser.open_fake_page(html)

    # Test StatefulBrowser.links url_regex argument
    assert browser.links(url_regex="bl") == expected
    assert browser.links(url_regex="bluish") == []

    # Test StatefulBrowser.links link_text argument
    assert browser.links(link_text="A Blue Link") == expected
    assert browser.links(link_text="Blue") == []

    # Test StatefulBrowser.links kwargs passed to BeautifulSoup.find_all
    assert browser.links(string=re.compile('Blue')) == expected
    assert browser.links(class_="bluelink") == expected
    assert browser.links(id="blue_link") == expected
    assert browser.links(id="blue") == []

    # Test returning a non-singleton
    two_links = browser.links(id=re.compile('_link'))
    assert len(two_links) == 2
    assert two_links == BeautifulSoup(html, "lxml").find_all('a') 

# 需要导入模块: import mechanicalsoup [as 别名]
# 或者: from mechanicalsoup import StatefulBrowser [as 别名]
def test_download_link(httpbin):
    """Test downloading the contents of a link to file."""
    browser = mechanicalsoup.StatefulBrowser()
    open_legacy_httpbin(browser, httpbin)
    tmpdir = tempfile.mkdtemp()
    tmpfile = tmpdir + '/nosuchfile.png'
    current_url = browser.url
    current_page = browser.page
    response = browser.download_link(file=tmpfile, link='image/png')

    # Check that the browser state has not changed
    assert browser.url == current_url
    assert browser.page == current_page

    # Check that the file was downloaded
    assert os.path.isfile(tmpfile)
    assert file_get_contents(tmpfile) == response.content
    # Check that we actually downloaded a PNG file
    assert response.content[:4] == b'\x89PNG' 

# 需要导入模块: import mechanicalsoup [as 别名]
# 或者: from mechanicalsoup import StatefulBrowser [as 别名]
def test_download_link_to_existing_file(httpbin):
    """Test downloading the contents of a link to an existing file."""
    browser = mechanicalsoup.StatefulBrowser()
    open_legacy_httpbin(browser, httpbin)
    tmpdir = tempfile.mkdtemp()
    tmpfile = tmpdir + '/existing.png'
    with open(tmpfile, "w") as f:
        f.write("initial content")
    current_url = browser.url
    current_page = browser.page
    response = browser.download_link('image/png', tmpfile)

    # Check that the browser state has not changed
    assert browser.url == current_url
    assert browser.page == current_page

    # Check that the file was downloaded
    assert os.path.isfile(tmpfile)
    assert file_get_contents(tmpfile) == response.content
    # Check that we actually downloaded a PNG file
    assert response.content[:4] == b'\x89PNG' 

# 需要导入模块: import mechanicalsoup [as 别名]
# 或者: from mechanicalsoup import StatefulBrowser [as 别名]
def test_download_link_404(httpbin):
    """Test downloading the contents of a broken link."""
    browser = mechanicalsoup.StatefulBrowser(raise_on_404=True)
    browser.open_fake_page('<a href="/no-such-page-404">Link</a>',
                           url=httpbin.url)
    tmpdir = tempfile.mkdtemp()
    tmpfile = tmpdir + '/nosuchfile.txt'
    current_url = browser.url
    current_page = browser.page
    with pytest.raises(mechanicalsoup.LinkNotFoundError):
        browser.download_link(file=tmpfile, link_text='Link')

    # Check that the browser state has not changed
    assert browser.url == current_url
    assert browser.page == current_page

    # Check that the file was not downloaded
    assert not os.path.exists(tmpfile) 

# 需要导入模块: import mechanicalsoup [as 别名]
# 或者: from mechanicalsoup import StatefulBrowser [as 别名]
def test_download_link_referer(httpbin):
    """Test downloading the contents of a link to file."""
    browser = mechanicalsoup.StatefulBrowser()
    ref = httpbin + "/my-referer"
    browser.open_fake_page('<a href="/headers">Link</a>',
                           url=ref)
    tmpfile = tempfile.NamedTemporaryFile()
    current_url = browser.url
    current_page = browser.page
    browser.download_link(file=tmpfile.name, link_text='Link')

    # Check that the browser state has not changed
    assert browser.url == current_url
    assert browser.page == current_page

    # Check that the file was downloaded
    with open(tmpfile.name) as f:
        json_data = json.load(f)
    headers = json_data["headers"]
    assert headers["Referer"] == ref 

# 需要导入模块: import mechanicalsoup [as 别名]
# 或者: from mechanicalsoup import StatefulBrowser [as 别名]
def authorize(auth_url, user, password):
    browser = mechanicalsoup.StatefulBrowser(raise_on_404=True)
    logger.info("opening auth_url")
    response = browser.open(auth_url)
    assert(response.ok)
    browser.select_form('form[action="/vpn-user-portal/_form/auth/verify"]')
    browser["userName"] = user
    browser["userPass"] = password
    logger.info("logging in")
    response = browser.submit_selected()
    assert(response.ok)
    form = browser.select_form()
    if 'action' in form.form.attrs and form.form.attrs['action'] == '/vpn-user-portal/_two_factor/auth/verify/totp':
        raise EduvpnAuthException("otp enabled")
    assert(urlparse(browser.get_url()).path == "/vpn-user-portal/_oauth/authorize")  # make sure is the right page
    form = browser.select_form()
    form.form.select('button[value="yes"]')
    logger.info("authorising app")
    response = browser.submit_selected()
    assert(response.ok) 

# 需要导入模块: import mechanicalsoup [as 别名]
# 或者: from mechanicalsoup import StatefulBrowser [as 别名]
def prepare_mock_browser(scheme='mock'):
    mock = requests_mock.Adapter()
    browser = mechanicalsoup.StatefulBrowser(requests_adapters={scheme: mock})

    return browser, mock 

# 需要导入模块: import mechanicalsoup [as 别名]
# 或者: from mechanicalsoup import StatefulBrowser [as 别名]
def test_choose_submit_fail(select_name):
    browser = mechanicalsoup.StatefulBrowser()
    browser.open_fake_page(choose_submit_fail_form)
    form = browser.select_form('#choose-submit-form')
    if select_name['fails']:
        with pytest.raises(mechanicalsoup.utils.LinkNotFoundError):
            form.choose_submit(select_name['name'])
    else:
        form.choose_submit(select_name['name']) 

# 需要导入模块: import mechanicalsoup [as 别名]
# 或者: from mechanicalsoup import StatefulBrowser [as 别名]
def test_choose_submit_multiple_match():
    browser = mechanicalsoup.StatefulBrowser()
    browser.open_fake_page(choose_submit_multiple_match_form)
    form = browser.select_form('#choose-submit-form')
    with pytest.raises(mechanicalsoup.utils.LinkNotFoundError):
        form.choose_submit('test_submit') 

# 需要导入模块: import mechanicalsoup [as 别名]
# 或者: from mechanicalsoup import StatefulBrowser [as 别名]
def test_form_set_radio_checkbox(capsys):
    browser = mechanicalsoup.StatefulBrowser()
    browser.open_fake_page(page_with_various_fields,
                           url="http://example.com/invalid/")
    form = browser.select_form("form")
    form.set_radio({"size": "small"})
    form.set_checkbox({"topping": "cheese"})
    browser.form.print_summary()
    out, err = capsys.readouterr()
    # Different versions of bs4 show either <input></input> or
    # <input/>. Normalize before comparing.
    out = out.replace('></input>', '/>')
    assert out == """<input name="foo"/>
<textarea name="bar"></textarea>
<select name="entree">
<option selected="selected" value="tofu">Tofu Stir Fry</option>
<option value="curry">Red Curry</option>
<option value="tempeh">Tempeh Tacos</option>
</select>
<input name="topping" type="checkbox" value="bacon"/>
<input checked="" name="topping" type="Checkbox" value="cheese"/>
<input name="topping" type="checkbox" value="onion"/>
<input name="topping" type="checkbox" value="mushroom"/>
<input checked="" name="size" type="Radio" value="small"/>
<input name="size" type="radio" value="medium"/>
<input name="size" type="radio" value="large"/>
<button name="action" value="cancel">Cancel</button>
<input type="submit" value="Select"/>
"""
    assert err == "" 

# 需要导入模块: import mechanicalsoup [as 别名]
# 或者: from mechanicalsoup import StatefulBrowser [as 别名]
def test_form_print_summary(capsys):
    browser = mechanicalsoup.StatefulBrowser()
    browser.open_fake_page(page_with_various_fields,
                           url="http://example.com/invalid/")
    browser.select_form("form")
    browser.form.print_summary()
    out, err = capsys.readouterr()
    # Different versions of bs4 show either <input></input> or
    # <input/>. Normalize before comparing.
    out = out.replace('></input>', '/>')
    assert out == """<input name="foo"/>
<textarea name="bar"></textarea>
<select name="entree">
<option selected="selected" value="tofu">Tofu Stir Fry</option>
<option value="curry">Red Curry</option>
<option value="tempeh">Tempeh Tacos</option>
</select>
<input name="topping" type="checkbox" value="bacon"/>
<input checked="" name="topping" type="Checkbox" value="cheese"/>
<input checked="" name="topping" type="checkbox" value="onion"/>
<input name="topping" type="checkbox" value="mushroom"/>
<input name="size" type="Radio" value="small"/>
<input name="size" type="radio" value="medium"/>
<input name="size" type="radio" value="large"/>
<button name="action" value="cancel">Cancel</button>
<input type="submit" value="Select"/>
"""
    assert err == "" 

# 需要导入模块: import mechanicalsoup [as 别名]
# 或者: from mechanicalsoup import StatefulBrowser [as 别名]
def test_issue180():
    """Test that a KeyError is not raised when Form.choose_submit is called
    on a form where a submit element is missing its name-attribute."""
    browser = mechanicalsoup.StatefulBrowser()
    html = '''
<form>
  <input type="submit" value="Invalid" />
  <input type="submit" name="valid" value="Valid" />
</form>
'''
    browser.open_fake_page(html)
    form = browser.select_form()
    with pytest.raises(mechanicalsoup.utils.LinkNotFoundError):
        form.choose_submit('not_found')