Python Article.create_articles方法代码示例

# 需要导入模块: from amcat.models import Article [as 别名]
# 或者: from amcat.models.Article import create_articles [as 别名]
    def test_aggregate(self):
        """Can we make tables per medium/date interval?"""
        from amcat.models import Article
        m1 = amcattest.create_test_medium(name="De Nep-Krant")
        m2, m3 = [amcattest.create_test_medium() for _ in range(2)]
        s1 = amcattest.create_test_set()
        s2 = amcattest.create_test_set()
        unused = amcattest.create_test_article(text='aap noot mies', medium=m3, articleset=s2)
        a = amcattest.create_test_article(text='aap noot mies', medium=m1, date='2001-01-01', create=False)
        b = amcattest.create_test_article(text='noot mies wim zus', medium=m2, date='2001-02-01', create=False)
        c = amcattest.create_test_article(text='mies bla bla bla wim zus jet', medium=m2, date='2002-01-01', create=False)
        d = amcattest.create_test_article(text='noot mies wim zus', medium=m2, date='2001-02-03', create=False)

        Article.create_articles([a,b,c,d], articleset=s1, check_duplicate=False, create_id=True)
        ES().flush()

        self.assertEqual(dict(ES().aggregate_query(filters=dict(sets=s1.id), group_by="mediumid")),
                         {m1.id : 1, m2.id : 3})

        self.assertEqual(dict(ES().aggregate_query(filters=dict(sets=s1.id), group_by="date", date_interval="year")),
                         {datetime(2001,1,1) : 3, datetime(2002,1,1) : 1})

        self.assertEqual(dict(ES().aggregate_query(filters=dict(sets=s1.id), group_by="date", date_interval="month")),
                         {datetime(2001,1,1) : 1, datetime(2002,1,1) : 1, datetime(2001,2,1) : 2})

        # set statistics
        stats = ES().statistics(filters=dict(sets=s1.id))
        self.assertEqual(stats.n, 4)
        self.assertEqual(stats.start_date, datetime(2001,1,1))
        self.assertEqual(stats.end_date, datetime(2002,1,1))

        # media list
        self.assertEqual(set(ES().list_media(filters=dict(sets=s1.id))),
                         {m1.id, m2.id})

# 需要导入模块: from amcat.models import Article [as 别名]
# 或者: from amcat.models.Article import create_articles [as 别名]
    def test_get_articles(self):
        from amcat.models import Sentence
        _get_articles = lambda a,s : list(get_articles(a,s))

        # Should raise exception if sentences not in article
        article, sentences = self.create_test_sentences()
        s1 = Sentence.objects.filter(id=amcattest.create_test_sentence().id)
        self.assertRaises(ValueError, _get_articles, article, s1)

        # Should raise an exception if we try to split on headline
        self.assertRaises(ValueError, _get_articles, article, sentences.filter(parnr=1))

        # Should return a "copy", with byline in "text" property
        arts = _get_articles(article, Sentence.objects.none())
        Article.create_articles(arts)

        self.assertEquals(len(arts), 1)
        sbd.create_sentences(arts[0])

        self.assertEquals(
            [s.sentence for s in sentences[1:]],
            [s.sentence for s in arts[0].sentences.all()[1:]]
        )

        self.assertTrue("foo" in arts[0].text)

        # Should be able to split on byline
        self.assertEquals(2, len(_get_articles(article, sentences[1:2])))
        a, b = _get_articles(article, sentences[4:5])

        # Check if text on splitted articles contains expected
        self.assertTrue("Einde" not in a.text)
        self.assertTrue("Einde" in b.text)

# 需要导入模块: from amcat.models import Article [as 别名]
# 或者: from amcat.models.Article import create_articles [as 别名]
 def test_create(self):
     """Can we create/store/index an article object?"""
     a = amcattest.create_test_article(create=False, date='2010-12-31', headline=u'\ua000abcd\u07b4')
     Article.create_articles([a], create_id=True)
     db_a = Article.objects.get(pk=a.id)
     amcates.ES().flush()
     es_a = list(amcates.ES().query(filters={'ids': [a.id]}, fields=["date", "headline"]))[0]
     self.assertEqual(a.headline, db_a.headline)
     self.assertEqual(a.headline, es_a.headline)
     self.assertEqual('2010-12-31T00:00:00', db_a.date.isoformat())
     self.assertEqual('2010-12-31T00:00:00', es_a.date.isoformat())

# 需要导入模块: from amcat.models import Article [as 别名]
# 或者: from amcat.models.Article import create_articles [as 别名]
 def test_create_order(self):
     """Is insert order preserved in id order?"""
     articles = [amcattest.create_test_article(create=False) for _i in range(25)]
     random.shuffle(articles)
     Article.create_articles(articles)
     ids = [a.id for a in articles]
     # is order preserved?
     self.assertEqual(ids, sorted(ids))
     # do the right articles have the right title?
     for saved in articles:
         indb = Article.objects.get(pk=saved.id)
         self.assertEqual(indb.title, saved.title)

# 需要导入模块: from amcat.models import Article [as 别名]
# 或者: from amcat.models.Article import create_articles [as 别名]
    def create(self, validated_data):
        articleset = self.get_articleset()

        if 'id' in validated_data:
            _check_read_access(self.context['request'].user, [validated_data['id']])
            article = Article.objects.get(pk=validated_data['id'])
            articleset.add_articles([article])
        else:
            article = json_to_article(validated_data, articleset.project)
            Article.create_articles([article], articleset=articleset)

        return article

# 需要导入模块: from amcat.models import Article [as 别名]
# 或者: from amcat.models.Article import create_articles [as 别名]
    def setup(self):
        s1 = amcattest.create_test_set()
        s2 = amcattest.create_test_set()
        a = amcattest.create_test_article(text='aap noot mies', title='m1', date='2001-01-01', create=False)
        b = amcattest.create_test_article(text='noot mies wim zus', title='m2', date='2001-02-01', create=False)
        c = amcattest.create_test_article(text='mies bla bla bla wim zus jet', title='m2', date='2002-01-01', create=False)
        d = amcattest.create_test_article(text='noot mies wim zus', title='m2', date='2001-02-03', create=False)
        e = amcattest.create_test_article(text='aap noot mies', title='m3', articleset=s2)

        Article.create_articles([a, b, c, d], articleset=s1)
        ES().refresh()
        return s1, s2, a, b, c, d, e

# 需要导入模块: from amcat.models import Article [as 别名]
# 或者: from amcat.models.Article import create_articles [as 别名]
 def test_create(self):
     """Can we create/store/index an article object?"""
     a = amcattest.create_test_article(create=False, date='2010-12-31', title=u'\ua000abcd\u07b4')
     Article.create_articles([a])
     db_a = Article.objects.get(pk=a.id)
     amcates.ES().refresh()
     es_a = list(amcates.ES().query(filters={'ids': [a.id]}, _source=["date", "title", "hash"]))[0]
     self.assertEqual(a.hash, db_a.hash)
     self.assertEqual(a.hash, es_a.hash)
     self.assertEqual(a.title, db_a.title)
     self.assertEqual(a.title, es_a.title)
     self.assertEqual('2010-12-31T00:00:00', db_a.date.isoformat())
     self.assertEqual('2010-12-31T00:00:00', es_a.date.isoformat())

# 需要导入模块: from amcat.models import Article [as 别名]
# 或者: from amcat.models.Article import create_articles [as 别名]
    def test_query_all(self):
        """Test that query_all works"""
        from amcat.models import Article
        arts = [amcattest.create_test_article(create=False) for _ in range(20)]
        s = amcattest.create_test_set()
        Article.create_articles(arts, articleset=s, check_duplicate=False, create_id=True)
        ES().flush()

        r = ES().query(filters=dict(sets=s.id), size=10)
        self.assertEqual(len(list(r)), 10)

        r = ES().query_all(filters=dict(sets=s.id), size=10)
        self.assertEqual(len(list(r)), len(arts))

# 需要导入模块: from amcat.models import Article [as 别名]
# 或者: from amcat.models.Article import create_articles [as 别名]
    def setup(self):
        m1 = amcattest.create_test_medium(name="De Nep-Krant")
        m2, m3 = [amcattest.create_test_medium() for _ in range(2)]
        s1 = amcattest.create_test_set()
        s2 = amcattest.create_test_set()
        a = amcattest.create_test_article(text='aap noot mies', medium=m1, date='2001-01-01', create=False)
        b = amcattest.create_test_article(text='noot mies wim zus', medium=m2, date='2001-02-01', create=False)
        c = amcattest.create_test_article(text='mies bla bla bla wim zus jet', medium=m2, date='2002-01-01', create=False)
        d = amcattest.create_test_article(text='noot mies wim zus', medium=m2, date='2001-02-03', create=False)
        e = amcattest.create_test_article(text='aap noot mies', medium=m3, articleset=s2)

        Article.create_articles([a, b, c, d], articleset=s1)
        ES().flush()
        return m1, m2, m3, s1, s2, a, b, c, d, e

# 需要导入模块: from amcat.models import Article [as 别名]
# 或者: from amcat.models.Article import create_articles [as 别名]
    def test_add_many(self):
        """Can we add a large number of articles from one set to another?"""
        s = amcattest.create_test_set()
        s2 = amcattest.create_test_set()
        p = amcattest.create_test_project()

        arts = [amcattest.create_test_article(project=p, create=False) for _x in range(1213)]
        Article.create_articles(arts, s)
        ES().refresh()
        self.assertEqual(len(arts), s.get_count())
        s2.add_articles(arts)
        ES().refresh()
        self.assertEqual(len(arts), s2.get_count())
        print(s2.get_count())

# 需要导入模块: from amcat.models import Article [as 别名]
# 或者: from amcat.models.Article import create_articles [as 别名]
    def test_add_many(self):
        """Can we add a large number of articles from one set to another?"""
        s = amcattest.create_test_set()
        s2 = amcattest.create_test_set()
        m = amcattest.create_test_medium()
        p = amcattest.create_test_project()

        arts = [amcattest.create_test_article(project=p, medium=m, create=False) for _x in range(1213)]
        Article.create_articles(arts, s, create_id=True)
        ES().flush()
        self.assertEqual(len(arts), s.get_count())
        s2.add_articles(arts, monitor=ProgressMonitor())
        ES().flush()
        self.assertEqual(len(arts), s2.get_count())
        print(s2.get_count())

# 需要导入模块: from amcat.models import Article [as 别名]
# 或者: from amcat.models.Article import create_articles [as 别名]
    def run(self, scraper):
        try:
            units = list(scraper._get_units())
        except Exception as e:
            self.errors.append(ScrapeError(None,None,e))
            log.exception("scraper._get_units failed")
            return self.articles

        for i, unit in enumerate(units):
            try:
                articles = list(scraper._scrape_unit(unit))
            except Exception as e:
                log.exception("scraper._scrape_unit failed")
                self.errors.append(ScrapeError(i,unit,e))
                continue
            self.articles += articles

        for article in self.articles:
            _set_default(article, 'project', scraper.project)

        try:
            articles, errors = Article.create_articles(self.articles, scraper.articleset)
            self.saved_article_ids = {getattr(a, "duplicate_of", a.id) for a in self.articles}
            for e in errors:
                self.errors.append(ScrapeError(None,None,e))
        except Exception as e:
            self.errors.append(ScrapeError(None,None,e))
	    print e
            log.exception("scraper._get_units failed")

        return self.saved_article_ids

# 需要导入模块: from amcat.models import Article [as 别名]
# 或者: from amcat.models.Article import create_articles [as 别名]
    def test_list_media(self):
        """Test that list media works for more than 10 media"""
        from amcat.models import Article
        media =  [amcattest.create_test_medium() for _ in range(20)]
        arts = [amcattest.create_test_article(medium=m, create=False) for m in media]

        s1 = amcattest.create_test_set()
        Article.create_articles(arts[:5], articleset=s1, check_duplicate=False, create_id=True)
        ES().flush()
        self.assertEqual(set(s1.get_mediums()), set(media[:5]))

        s2 = amcattest.create_test_set(project=s1.project)
        Article.create_articles(arts[5:], articleset=s2, check_duplicate=False, create_id=True)
        ES().flush()
        self.assertEqual(set(s2.get_mediums()), set(media[5:]))

        self.assertEqual(set(s1.project.get_mediums()), set(media))

# 需要导入模块: from amcat.models import Article [as 别名]
# 或者: from amcat.models.Article import create_articles [as 别名]
 def create_articles(batch):
     for a in batch:
         a['oldid_int'] = a.pop('old_id')
         if a['text'] == '': a['text'] = '-'
         if a['title'] == '': a['title'] = '-'
     articles = Article.create_articles([Article(project_id=self.status.project.id, **a) for a in batch])
     self.status.articles.update({a.get_property('oldid_int'): a.id for a in articles})
     return articles

# 需要导入模块: from amcat.models import Article [as 别名]
# 或者: from amcat.models.Article import create_articles [as 别名]
def handle_split(form, project, article, sentences):
    articles = list(get_articles(article, sentences))

    # We won't use bulk_create yet, as it bypasses save() and doesn't
    # insert ids
    Article.create_articles(articles)
    for art in articles:
        sbd.get_or_create_sentences(art)

    if not form.is_valid():
        raise ValueError("Form invalid: {form.errors}".format(**locals()))

    # Context variables for template
    form_data = form.cleaned_data
    all_sets = list(project.all_articlesets().filter(articles=article))

    # Add splitted articles to existing sets
    for aset in form_data["add_splitted_to_sets"]:
        aset.add_articles(articles)

    # Add splitted articles to sets wherin the original article live{d,s}
    if form_data["add_splitted_to_all"]:
        asets = project.all_articlesets().filter(articles=article).only("id")
        for aset in asets:
            aset.add_articles(articles)

    if form_data["remove_from_sets"]:
        for aset in form_data["remove_from_sets"]:
            aset.remove_articles([article])

    if form_data["remove_from_all_sets"]:
        for aset in ArticleSet.objects.filter(project=project, articles=article).distinct():
            aset.remove_articles([article])

    if form_data["add_splitted_to_new_set"]:
        new_splitted_set = ArticleSet.create_set(project, form_data["add_splitted_to_new_set"], articles)

    if form_data["add_to_sets"]:
        for articleset in form_data["add_to_sets"]:
            articleset.add_articles([article])

    if form_data["add_to_new_set"]:
        new_set = ArticleSet.create_set(project, form_data["add_to_new_set"], [article])

    return locals()