本文整理汇总了Python中models.SyndicatedPost类的典型用法代码示例。如果您正苦于以下问题：Python SyndicatedPost类的具体用法？Python SyndicatedPost怎么用？Python SyndicatedPost使用的例子？那么恭喜您, 这里精选的类代码示例或许可以为您提供帮助。
在下文中一共展示了SyndicatedPost类的15个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: test_syndication_url_in_hfeed
def test_syndication_url_in_hfeed(self):
"""Like test_single_post, but because the syndication URL is given in
the h-feed we skip fetching the permalink. New behavior as of
2014-11-08
"""
self.activity['object']['upstreamDuplicates'] = ['existing uD']
# silo domain is fa.ke
self.expect_requests_get('http://author', """
<html class="h-feed">
<div class="h-entry">
<a class="u-url" href="http://author/post/permalink"></a>
<a class="u-syndication" href="http://fa.ke/post/url">
</div>
</html>""")
self.mox.ReplayAll()
logging.debug('Original post discovery %s -> %s', self.source, self.activity)
original_post_discovery.discover(self.source, self.activity)
# upstreamDuplicates = 1 original + 1 discovered
self.assertEquals(['existing uD', 'http://author/post/permalink'],
self.activity['object']['upstreamDuplicates'])
origurls = [r.original for r in SyndicatedPost.query(ancestor=self.source.key)]
self.assertEquals([u'http://author/post/permalink'], origurls)
# for now only syndicated posts belonging to this source are stored
syndurls = list(r.syndication for r
in SyndicatedPost.query(ancestor=self.source.key))
self.assertEquals([u'https://fa.ke/post/url'], syndurls)示例2: test_multiple_refetches
def test_multiple_refetches(self):
"""Ensure that multiple refetches of the same post (with and without
u-syndication) does not generate duplicate blank entries in the
database. See https://github.com/snarfed/bridgy/issues/259 for details
"""
self.activities[0]['object'].update({
'content': 'post content without backlinks',
'url': 'https://fa.ke/post/url',
})
hfeed = """<html class="h-feed">
<a class="h-entry" href="/permalink"></a>
</html>"""
unsyndicated = """<html class="h-entry">
<a class="u-url" href="/permalink"></a>
</html>"""
syndicated = """<html class="h-entry">
<a class="u-url" href="/permalink"></a>
<a class="u-syndication" href="https://fa.ke/post/url"></a>
</html>"""
# first attempt, no syndication url yet
self.expect_requests_get('http://author', hfeed)
self.expect_requests_get('http://author/permalink', unsyndicated)
# refetch, still no syndication url
self.expect_requests_get('http://author', hfeed)
self.expect_requests_get('http://author/permalink', unsyndicated)
# second refetch, has a syndication url this time
self.expect_requests_get('http://author', hfeed)
self.expect_requests_get('http://author/permalink', syndicated)
self.mox.ReplayAll()
original_post_discovery.discover(self.source, self.activities[0])
original_post_discovery.refetch(self.source)
relations = list(
SyndicatedPost.query(
SyndicatedPost.original == 'http://author/permalink',
ancestor=self.source.key).fetch())
self.assertEquals(1, len(relations))
self.assertEquals('http://author/permalink', relations[0].original)
self.assertIsNone(relations[0].syndication)
original_post_discovery.refetch(self.source)
relations = list(
SyndicatedPost.query(
SyndicatedPost.original == 'http://author/permalink',
ancestor=self.source.key).fetch())
self.assertEquals(1, len(relations))
self.assertEquals('http://author/permalink', relations[0].original)
self.assertEquals('https://fa.ke/post/url', relations[0].syndication)示例3: test_multiple_rel_feeds
def test_multiple_rel_feeds(self):
"""Make sure that we follow all rel=feed links, e.g. if notes and
articles are in separate feeds."""
self.expect_requests_get('http://author', """
<html>
<head>
<link rel="feed" href="/articles" type="text/html">
<link rel="feed" href="/notes" type="text/html">
</head>
</html>""")
# fetches all feeds first
self.expect_requests_get('http://author/articles', """
<html class="h-feed">
<article class="h-entry">
<a class="u-url" href="/article-permalink"></a>
</article>
</html>""").InAnyOrder('feed')
self.expect_requests_get('http://author/notes', """
<html class="h-feed">
<article class="h-entry">
<a class="u-url" href="/note-permalink"></a>
</article>
</html>""").InAnyOrder('feed')
# then the permalinks (in any order since they are hashed to
# remove duplicates)
self.expect_requests_get('http://author/article-permalink', """
<html class="h-entry">
<a class="u-url" href="/article-permalink"></a>
<a class="u-syndication" href="https://fa.ke/article"></a>
</html>""").InAnyOrder('permalink')
self.expect_requests_get('http://author/note-permalink', """
<html class="h-entry">
<a class="u-url" href="/note-permalink"></a>
<a class="u-syndication" href="https://fa.ke/note"></a>
</html>""").InAnyOrder('permalink')
self.mox.ReplayAll()
original_post_discovery.discover(self.source, self.activity)
note_rels = SyndicatedPost.query(
SyndicatedPost.original == 'http://author/note-permalink',
ancestor=self.source.key).fetch()
self.assertEqual(1, len(note_rels))
self.assertEqual('https://fa.ke/note', note_rels[0].syndication)
article_rels = SyndicatedPost.query(
SyndicatedPost.original == 'http://author/article-permalink',
ancestor=self.source.key).fetch()
self.assertEqual(1, len(article_rels))
self.assertEqual('https://fa.ke/article', article_rels[0].syndication)示例4: _posse_post_discovery
def _posse_post_discovery(source, activity, syndication_url, fetch_hfeed,
already_fetched_hfeeds):
"""Performs the actual meat of the posse-post-discover.
Args:
source: :class:`models.Source` subclass
activity: activity dict
syndication_url: url of the syndicated copy for which we are
trying to find an original
fetch_hfeed: boolean, whether or not to fetch and parse the
author's feed if we don't have a previously stored
relationship
already_fetched_hfeeds: set, URLs we've already fetched in a
previous iteration
Return:
sequence of string original post urls, possibly empty
"""
logging.info('starting posse post discovery with syndicated %s',
syndication_url)
relationships = SyndicatedPost.query(
SyndicatedPost.syndication == syndication_url,
ancestor=source.key).fetch()
if not relationships and fetch_hfeed:
# a syndicated post we haven't seen before! fetch the author's URLs to see
# if we can find it.
#
# TODO: Consider using the actor's url, with get_author_urls() as the
# fallback in the future to support content from non-Bridgy users.
results = {}
for url in _get_author_urls(source):
if url not in already_fetched_hfeeds:
results.update(_process_author(source, url))
already_fetched_hfeeds.add(url)
else:
logging.debug('skipping %s, already fetched this round', url)
relationships = results.get(syndication_url, [])
if not relationships:
# No relationships were found. Remember that we've seen this
# syndicated post to avoid reprocessing it every time
logging.debug('posse post discovery found no relationship for %s',
syndication_url)
if fetch_hfeed:
SyndicatedPost.insert_syndication_blank(source, syndication_url)
originals = [r.original for r in relationships if r.original]
if originals:
logging.debug('posse post discovery found relationship(s) %s -> %s',
syndication_url, originals)
return originals示例5: test_query_by_original_url
def test_query_by_original_url(self):
"""Simply testing the query helper"""
r = SyndicatedPost.query_by_original(
self.source, 'http://original/post/url')
self.assertIsNotNone(r)
self.assertEquals('http://silo/post/url', r.syndication)
r = SyndicatedPost.query_by_original(
self.source, 'http://original/no-syndication')
self.assertIsNotNone(r)
self.assertIsNone(r.syndication)示例6: test_get_or_insert_by_syndication_do_not_duplicate_blanks
def test_get_or_insert_by_syndication_do_not_duplicate_blanks(self):
"""Make sure we don't insert duplicate blank entries"""
SyndicatedPost.insert_syndication_blank(
self.source, 'http://silo/no-original')
# make sure there's only one in the DB
rs = SyndicatedPost.query(
SyndicatedPost.syndication == 'http://silo/no-original',
ancestor=self.source.key
).fetch()
self.assertItemsEqual([None], [rel.original for rel in rs])示例7: test_refetch_multiple_responses_same_activity
def test_refetch_multiple_responses_same_activity(self):
"""Ensure that refetching a post that has several replies does not
generate duplicate original -> None blank entries in the
database. See https://github.com/snarfed/bridgy/issues/259 for
details
"""
source = self.sources[0]
source.domain_urls = ['http://author']
for activity in self.activities:
activity['object']['content'] = 'post content without backlinks'
activity['object']['url'] = 'https://fa.ke/post/url'
author_feed = """
<html class="h-feed">
<div class="h-entry">
<a class="u-url" href="http://author/post/permalink"></a>
</div>
</html>"""
author_entry = """
<html class="h-entry">
<a class="u-url" href="http://author/post/permalink"></a>
</html>"""
# original
self.expect_requests_get('http://author', author_feed)
self.expect_requests_get('http://author/post/permalink', author_entry)
# refetch
self.expect_requests_get('http://author', author_feed)
self.expect_requests_get('http://author/post/permalink', author_entry)
self.mox.ReplayAll()
for activity in self.activities:
original_post_discovery.discover(source, activity)
original_post_discovery.refetch(source)
rels_by_original = list(
SyndicatedPost.query(SyndicatedPost.original == 'http://author/post/permalink',
ancestor=source.key).fetch())
self.assertEquals(1, len(rels_by_original))
self.assertIsNone(rels_by_original[0].syndication)
rels_by_syndication = list(
SyndicatedPost.query(SyndicatedPost.syndication == 'https://fa.ke/post/url',
ancestor=source.key).fetch())
self.assertEquals(1, len(rels_by_syndication))
self.assertIsNone(rels_by_syndication[0].original)示例8: _posse_post_discovery
def _posse_post_discovery(source, activity, syndication_url, fetch_hfeed):
"""Performs the actual meat of the posse-post-discover.
Args:
source: models.Source subclass
activity: activity dict
syndication_url: url of the syndicated copy for which we are
trying to find an original
fetch_hfeed: boolean, whether or not to fetch and parse the
author's feed if we don't have a previously stored
relationship.
Return:
the activity, updated with original post urls if any are found
"""
logging.info('starting posse post discovery with syndicated %s', syndication_url)
relationships = SyndicatedPost.query(
SyndicatedPost.syndication == syndication_url,
ancestor=source.key).fetch()
if not relationships and fetch_hfeed:
# a syndicated post we haven't seen before! fetch the author's URLs to see
# if we can find it.
#
# Use source.domain_urls for now; it seems more reliable than the
# activity.actor.url (which depends on getting the right data back from
# various APIs). Consider using the actor's url, with domain_urls as the
# fallback in the future to support content from non-Bridgy users.
results = {}
for url in source.get_author_urls():
results.update(_process_author(source, url))
relationships = results.get(syndication_url)
if not relationships:
# No relationships were found. Remember that we've seen this
# syndicated post to avoid reprocessing it every time
logging.debug('posse post discovery found no relationship for %s',
syndication_url)
if fetch_hfeed:
SyndicatedPost.insert_syndication_blank(source, syndication_url)
return activity
logging.debug('posse post discovery found relationship(s) %s -> %s',
syndication_url,
'; '.join(unicode(r.original) for r in relationships))
obj = activity.get('object') or activity
obj.setdefault('upstreamDuplicates', []).extend(
r.original for r in relationships if r.original)
return activity示例9: test_single_post
def test_single_post(self):
"""Test that original post discovery does the reverse lookup to scan
author's h-feed for rel=syndication links
"""
activity = self.activities[0]
activity['object'].update({
'content': 'post content without backlink',
'url': 'http://fa.ke/post/url',
'upstreamDuplicates': ['existing uD'],
})
# silo domain is fa.ke
source = self.sources[0]
source.domain_urls = ['http://author']
self.expect_requests_get(
'http://author', """
<html class="h-feed">
<div class="h-entry">
<a class="u-url" href="http://author/post/permalink"></a>
</div>
</html>""")
# syndicated to two places
self.expect_requests_get(
'http://author/post/permalink', """
<link rel="syndication" href="http://not.real/statuses/postid">
<link rel="syndication" href="http://fa.ke/post/url">
<div class="h-entry">
<a class="u-url" href="http://author/post/permalink"></a>
</div>""")
self.mox.ReplayAll()
logging.debug('Original post discovery %s -> %s', source, activity)
original_post_discovery.discover(source, activity)
# upstreamDuplicates = 1 original + 1 discovered
self.assertEquals(['existing uD', 'http://author/post/permalink'],
activity['object']['upstreamDuplicates'])
origurls = [
r.original for r in SyndicatedPost.query(ancestor=source.key)
]
self.assertEquals([u'http://author/post/permalink'], origurls)
# for now only syndicated posts belonging to this source are stored
syndurls = list(
r.syndication for r in SyndicatedPost.query(ancestor=source.key))
self.assertEquals([u'https://fa.ke/post/url'], syndurls)示例10: _posse_post_discovery
def _posse_post_discovery(source, activity, author_url, syndication_url,
fetch_hfeed):
"""Performs the actual meat of the posse-post-discover. It was split
out from discover() so that it can be done inside of a transaction.
Args:
source: models.Source subclass
activity: activity dict
author_url: author's url configured in their silo profile
syndication_url: url of the syndicated copy for which we are
trying to find an original
fetch_hfeed: boolean, whether or not to fetch and parse the
author's feed if we don't have a previously stored
relationship.
Return:
the activity, updated with original post urls if any are found
"""
logging.info(
'starting posse post discovery with author %s and syndicated %s',
author_url, syndication_url)
relationships = SyndicatedPost.query(
SyndicatedPost.syndication == syndication_url,
ancestor=source.key).fetch()
if not relationships and fetch_hfeed:
# a syndicated post we haven't seen before! fetch the author's
# h-feed to see if we can find it.
results = _process_author(source, author_url)
relationships = results.get(syndication_url)
if not relationships:
# No relationships were found. Remember that we've seen this
# syndicated post to avoid reprocessing it every time
logging.debug('posse post discovery found no relationship for %s',
syndication_url)
SyndicatedPost.insert_syndication_blank(source, syndication_url)
return activity
logging.debug('posse post discovery found relationship(s) %s -> %s',
syndication_url,
'; '.join(str(r.original) for r in relationships))
obj = activity.get('object') or activity
obj.setdefault('upstreamDuplicates', []).extend(
r.original for r in relationships if r.original)
return activity示例11: test_discover_url_site_post_syndication_links
def test_discover_url_site_post_syndication_links(self):
self.expect_requests_get('http://si.te/123', """
<div class="h-entry">
foo
<a class="u-syndication" href="http://fa.ke/222"></a>
<a class="u-syndication" href="http://other/silo"></a>
<a class="u-syndication" href="http://fa.ke/post/444"></a>
</div>""")
self.mox.ReplayAll()
self.assertEqual(0, SyndicatedPost.query().count())
self.check_discover('http://si.te/123',
'Discovering now. Refresh in a minute to see the results!')
self.assertItemsEqual([
{'https://fa.ke/222': 'http://si.te/123'},
{'https://fa.ke/post/444': 'http://si.te/123'},
], [{sp.syndication: sp.original} for sp in models.SyndicatedPost.query()])
tasks = self.taskqueue_stub.GetTasks('discover')
key = self.source.key.urlsafe()
self.assertEqual([
{'source_key': key, 'post_id': '222'},
{'source_key': key, 'post_id': '444'},
], [testutil.get_task_params(task) for task in tasks])
now = util.now_fn()
source = self.source.key.get()
self.assertEqual(now, source.last_syndication_url)示例12: test_insert_no_duplicates
def test_insert_no_duplicates(self):
"""Make sure we don't insert duplicate entries"""
r = SyndicatedPost.insert(
self.source, 'http://silo/post/url', 'http://original/post/url')
self.assertIsNotNone(r)
self.assertEqual('http://original/post/url', r.original)
# make sure there's only one in the DB
rs = SyndicatedPost.query(
SyndicatedPost.syndication == 'http://silo/post/url',
SyndicatedPost.original == 'http://original/post/url',
ancestor=self.source.key
).fetch()
self.assertEqual(1, len(rs))示例13: _process_syndication_urls
def _process_syndication_urls(source, permalink, syndication_urls):
"""Process a list of syndication URLs looking for one that matches the
current source. If one is found, stores a new SyndicatedPost in the
db.
Args:
source: a models.Source subclass
permalink: a string. the current h-entry permalink
syndication_urls: a collection of strings. the unfitered list
of syndication_urls
"""
results = {}
# save the results (or lack thereof) to the db, and put them in a
# map for immediate use
for syndication_url in syndication_urls:
# follow redirects to give us the canonical syndication url --
# gives the best chance of finding a match.
syndication_url = util.follow_redirects(syndication_url).url
# source-specific logic to standardize the URL. (e.g., replace facebook
# username with numeric id)
syndication_url = source.canonicalize_syndication_url(syndication_url)
# check that the syndicated url belongs to this source TODO save future
# lookups by saving results for other sources too (note: query the
# appropriate source subclass by author.domains, rather than
# author.domain_urls)
if util.domain_from_link(syndication_url) == source.AS_CLASS.DOMAIN:
logging.debug('saving discovered relationship %s -> %s',
syndication_url, permalink)
relationship = SyndicatedPost.insert(
source, syndication=syndication_url, original=permalink)
results.setdefault(syndication_url, []).append(relationship)
return results示例14: test_do_not_fetch_hfeed
def test_do_not_fetch_hfeed(self):
"""Confirms behavior of discover() when fetch_hfeed=False.
Discovery should only check the database for previously discovered matches.
It should not make any GET requests
"""
discover(self.source, self.activity, fetch_hfeed=False)
self.assertFalse(SyndicatedPost.query(ancestor=self.source.key).get())示例15: _test_failed_post_permalink_fetch
def _test_failed_post_permalink_fetch(self, raise_exception):
"""Make sure something reasonable happens when we're unable to fetch
the permalink of an entry linked in the h-feed
"""
source = self.sources[0]
source.domain_urls = ['http://author']
activity = self.activities[0]
activity['object']['url'] = 'https://fa.ke/post/url'
activity['object']['content'] = 'content without links'
self.expect_requests_get('http://author', """
<html class="h-feed">
<article class="h-entry">
<a class="u-url" href="nonexistent.html"></a>
</article>
</html>
""")
if raise_exception:
self.expect_requests_get('http://author/nonexistent.html').AndRaise(HTTPError())
else:
self.expect_requests_get('http://author/nonexistent.html', status_code=410)
self.mox.ReplayAll()
original_post_discovery.discover(source, activity)
# we should have saved placeholders to prevent us from trying the
# syndication url or permalink again
self.assert_equals(
set([('http://author/nonexistent.html', None), (None, 'https://fa.ke/post/url')]),
set((relationship.original, relationship.syndication)
for relationship in SyndicatedPost.query(ancestor=source.key)))