本文整理汇总了Python中readability.Document类的典型用法代码示例。如果您正苦于以下问题：Python Document类的具体用法？Python Document怎么用？Python Document使用的例子？那么恭喜您, 这里精选的类代码示例或许可以为您提供帮助。
在下文中一共展示了Document类的15个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: test_correct_cleanup
def test_correct_cleanup(self):
sample = """
<html>
<body>
<section>test section</section>
<article class="">
<p>Lot of text here.</p>
<div id="advertisement"><a href="link">Ad</a></div>
<p>More text is written here, and contains punctuation and dots.</p>
</article>
<aside id="comment1"/>
<div id="comment2">
<a href="asd">spam</a>
<a href="asd">spam</a>
<a href="asd">spam</a>
</div>
<div id="comment3"/>
<aside id="comment4">A small comment.</aside>
<div id="comment5"><p>The comment is also helpful, but it's
still not the correct item to be extracted.</p>
<p>It's even longer than the article itself!"</p></div>
</body>
</html>
"""
doc = Document(sample)
s = doc.summary()
#print(s)
assert('punctuation' in s)
assert(not 'comment' in s)
assert(not 'aside' in s)示例2: test_lxml_obj_result
def test_lxml_obj_result(self):
"""Feed Document with an lxml obj instead of an html string. Expect an lxml response"""
utf8_parser = lxml.html.HTMLParser(encoding='utf-8')
sample = lxml.html.document_fromstring(load_sample('nyt-article-video.sample.html'), parser=utf8_parser)
doc = Document(sample, url='http://nytimes.com/')
res = doc.summary()
self.assertFalse(isinstance(res, basestring))示例3: test_si_sample_html_partial
def test_si_sample_html_partial(self):
"""Using the si sample, make sure we can get the article alone."""
sample = load_sample('si-game.sample.html')
doc = Document(sample)
doc.parse(["summary"], html_partial=True)
res = doc.summary()
self.assertEqual('<div><h1>Tigers-R', res[0:17])示例4: get
def get(self):
url = self.get_argument("url", None)
# https://www.ifanr.com/1080409
doc = Webcache.find_one({'url': url}, {'_id': 0})
if doc:
self.res = dict(doc)
return self.write_json()
try:
sessions = requests.session()
sessions.headers[
'User-Agent'] = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.131 Safari/537.36'
response = sessions.get(url)
# response.encoding = 'utf-8' # TODO
response.encoding = get_charset(response)
doc = Document(response.text)
title = doc.title()
summary = doc.summary()
markdown = html2text.html2text(summary)
markdown = markdown.replace('-\n', '-')
markdown = markdown.strip()
res = {}
res['url'] = url
res['title'] = title
res['markdown'] = markdown
if title and markdown:
webcache = Webcache
webcache.new(res)
self.res = res
self.write_json()
except Exception as e:
print(e)示例5: test_si_sample_html_partial
def test_si_sample_html_partial(self):
"""Using the si sample, make sure we can get the article alone."""
sample = load_sample('si-game.sample.html')
doc = Document('http://sportsillustrated.cnn.com/baseball/mlb/gameflash/2012/04/16/40630_preview.html',
sample)
res = doc.get_clean_article()
self.assertEqual('<div><div class="', res[0:17])示例6: process_item
def process_item(self, article, spider):
doc = Document(article['text'])
article['text'] = strip_tags(doc.summary())
article['hash'] = hashlib.sha256(article['url']).hexdigest()
return article示例7: test_si_sample
def test_si_sample(self):
"""Using the si sample, load article with only opening body element"""
sample = load_sample('si-game.sample.html')
doc = Document(sample)
doc.parse(["summary"])
res = doc.summary()
self.assertEqual('<html><body><h1>Tigers-Roya', res[0:27])示例8: convert
def convert(link):
"""
use burify's readability implementation to transcode a web page
and return the transcoded page and images found in it
"""
if not link:
logger.error('Cannot transcode nothing!')
return None, None, None
try:
data = transcoder.prepare_link(link)
if data:
article = Document(data)
if article:
images, content = _collect_images(
article.summary(html_partial=False), link)
return article.short_title(), content, images
else:
logger.info('Burify cannot recognize the data')
return None, None, None
else:
logger.info('Cannot parse %s correctly' % link)
return None, None, None
except Exception as k:
logger.error('%s for %s' % (str(k), str(link)))
return None, None, None示例9: test_si_sample_html_partial
def test_si_sample_html_partial(self):
"""Using the si sample, make sure we can get the article alone."""
sample = load_sample("si-game.sample.html")
doc = Document(
sample, url="http://sportsillustrated.cnn.com/baseball/mlb/gameflash/2012/04/16/40630_preview.html"
)
res = doc.summary(enclose_with_html_tag=True)
self.assertEqual('<div><div class="', res[0:17])示例10: test_lazy_images
def test_lazy_images(self):
"""
Some sites use <img> elements with data-lazy-src elements pointing to the actual image.
"""
sample = load_sample('wired.sample.html')
doc = Document('http://www.wired.com/design/2014/01/will-influential-ui-design-minority-report/', sample)
article = doc.get_clean_article()
self.assertIn('<img src="http://www.wired.com/images_blogs/design/2014/01/her-joaquin-phoenix-41-660x371.jpg"', article)示例11: test_many_repeated_spaces
def test_many_repeated_spaces(self):
long_space = ' ' * 1000000
sample = '<html><body><p>foo' + long_space + '</p></body></html>'
doc = Document(sample)
s = doc.summary()
assert 'foo' in s示例12: test_si_sample
def test_si_sample(self):
"""Using the si sample, load article with only opening body element"""
sample = load_sample('si-game.sample.html')
doc = Document(
sample,
url='http://sportsillustrated.cnn.com/baseball/mlb/gameflash/2012/04/16/40630_preview.html')
res = doc.summary()
self.assertEqual('<html><body><div><div class', res[0:27])示例13: get
def get(self):
urls = self.get_query_arguments('url')
if urls and len(urls) == 1:
url = urls[0]
doc = Document(requests.get(url).text)
self.write(smartypants(doc.summary()))
self.write(STYLE)
else:
self.write("Please provide ?url=[your-url]")示例14: transform
def transform(self, row, chan):
row['response'] = resolve_future(row['response'])
doc = Document(row['response'].content)
row['title'] = doc.title()
summary = doc.summary()
row['text'] = html2text(summary, bodywidth=160).replace('****', '').strip()
yield row示例15: test_best_elem_is_root_and_passing
def test_best_elem_is_root_and_passing(self):
sample = (
'<html class="article" id="body">'
' <body>'
' <p>1234567890123456789012345</p>'
' </body>'
'</html>'
)
doc = Document(sample)
doc.summary()