Python models.Dataset类代码示例

    def get_context(self):
        context = super(OrganizationDetailView, self).get_context()

        can_edit = EditOrganizationPermission(self.organization)
        can_view = OrganizationPrivatePermission(self.organization)

        if self.organization.deleted and not can_view.can():
            abort(410)

        datasets = Dataset.objects(organization=self.organization).order_by('-temporal_coverage.end', '-metrics.reuses', '-metrics.followers').visible()
        reuses = Reuse.objects(organization=self.organization).order_by('-metrics.reuses', '-metrics.followers').visible()
        followers = (Follow.objects.followers(self.organization)
                                   .order_by('follower.fullname'))
        context.update({
            'reuses': reuses.paginate(1, self.page_size),
            'datasets': datasets.paginate(1, self.page_size),
            'followers': followers,
            'can_edit': can_edit,
            'can_view': can_view,
            'private_reuses': (
                list(Reuse.objects(organization=self.object).hidden())
                if can_view else []),
            'private_datasets': (
                list(Dataset.objects(organization=self.object).hidden())
                if can_view else []),
        })
        return context

 def from_organizations(self, user, *organizations):
     from udata.models import Dataset, Reuse  # Circular imports.
     Qs = db.Q()
     for dataset in Dataset.objects(owner=user).visible():
         Qs |= db.Q(subject=dataset)
     for org in organizations:
         for dataset in Dataset.objects(organization=org).visible():
             Qs |= db.Q(subject=dataset)
     for reuse in Reuse.objects.owned_by(*[user.id] + list(organizations)):
         Qs |= db.Q(subject=reuse)
     return self(Qs)

    def test_default(self):
        org = OrganizationFactory()
        source = HarvestSourceFactory(backend='factory', organization=org)
        with assert_emit(signals.before_harvest_job,
                         signals.after_harvest_job):
            self.action(source.slug)

        source.reload()
        assert len(HarvestJob.objects(source=source)) == 1

        job = source.get_last_job()

        assert job.status == 'done'
        assert job.errors == []
        assert job.started is not None
        assert job.ended is not None
        assert len(job.items) == COUNT

        for item in job.items:
            assert item.status == 'done'
            assert item.errors == []
            assert item.started is not None
            assert item.ended is not None
            assert item.dataset is not None

            dataset = item.dataset
            assert Dataset.objects(id=dataset.id).first() is not None
            assert dataset.organization == org
            assert 'harvest:remote_id' in dataset.extras
            assert 'harvest:last_update' in dataset.extras
            assert 'harvest:source_id' in dataset.extras

        assert len(HarvestJob.objects) == 1
        assert len(Dataset.objects) == COUNT

    def get(self, level):
        pipeline = [
            {'$project': {'territory': '$spatial.territories'}},
            {'$unwind': '$territory'},
            {'$match': {'territory.level': level}},
            {'$group': {'_id': '$territory.id', 'count': {'$sum': 1}}}
        ]
        features = []

        for row in Dataset.objects(spatial__territories__level=level).visible().aggregate(*pipeline):
            territory = Territory.objects.get(id=row['_id'])
            features.append({
                'id': str(territory.id),
                'type': 'Feature',
                'geometry': territory.geom,
                'properties': {
                    'name': territory.name,
                    'code': territory.code,
                    'level': territory.level,
                    'datasets': row['count']
                }
            })

        return {
            'type': 'FeatureCollection',
            'features': features
        }

    def test_attach_does_not_duplicate(self):
        attached_datasets = []
        for i in range(2):
            dataset = DatasetFactory.build()
            dataset.extras["harvest:domain"] = "test.org"
            dataset.extras["harvest:remote_id"] = str(i)
            dataset.save()
            attached_datasets.append(dataset)

        datasets = DatasetFactory.create_batch(3)

        with NamedTemporaryFile() as csvfile:
            writer = csv.DictWriter(csvfile, fieldnames=["local", "remote"], delimiter=b";", quotechar=b'"')

            writer.writeheader()
            for index, dataset in enumerate(datasets):
                writer.writerow({"local": str(dataset.id), "remote": str(index)})
            csvfile.flush()

            result = actions.attach("test.org", csvfile.name)

        dbcount = Dataset.objects(**{"extras__harvest:remote_id__exists": True}).count()
        self.assertEqual(result.success, len(datasets))
        self.assertEqual(dbcount, result.success)
        for index, dataset in enumerate(datasets):
            dataset.reload()
            self.assertEqual(dataset.extras["harvest:domain"], "test.org")
            self.assertEqual(dataset.extras["harvest:remote_id"], str(index))

    def get_context(self):
        context = super(OrganizationDetailView, self).get_context()

        org_id = str(self.organization.id)
        datasets, supplied_datasets, reuses = search.multiquery(
            search.SearchQuery(Dataset, sort='-created', organization=org_id, page_size=9),
            search.SearchQuery(Dataset, sort='-created', supplier=org_id, page_size=9),
            search.SearchQuery(Reuse, sort='-created', organization=org_id, page_size=9),

        )
        followers = FollowOrg.objects.followers(self.organization).order_by('follower.fullname')

        can_edit = EditOrganizationPermission(self.organization.id)
        context.update({
            'reuses': reuses,
            'datasets': datasets,
            'supplied_datasets': supplied_datasets,
            'followers': followers[:self.nb_followers],
            'can_edit': can_edit
        })
        if can_edit:
            context.update({
                'private_reuses': list(Reuse.objects(organization=self.object, private=True)),
                'private_datasets': list(Dataset.objects(organization=self.object, private=True)),
            })

        return context

 def get_value(self):
     ids = itertools.chain(*[
         [r.id for r in d.resources] for d in
         (Dataset.objects(organization=self.target).only('resources') or [])
     ])
     return int(Metrics.objects(object_id__in=ids, level='daily')
                       .sum('values.nb_uniq_visitors'))

    def get(self, level):
        '''List each zone for a given level with their datasets count'''
        level = GeoLevel.objects.get_or_404(id=level)
        features = []

        for zone in GeoZone.objects(level=level.id):
            # fetch nested levels IDs
            ids = GeoZone.objects(parents=zone.id).only('id').distinct('id')
            ids.append(zone.id)
            # Count datasets in zone
            nb_datasets = Dataset.objects(spatial__zones__in=ids).count()
            features.append({
                'id': zone.id,
                'type': 'Feature',
                'geometry': zone.geom,
                'properties': {
                    'name': _(zone.name),
                    'code': zone.code,
                    'level': zone.level,
                    'datasets': nb_datasets
                }
            })

        return {
            'type': 'FeatureCollection',
            'features': features
        }

def nec_mergitur():
    datasets = (Dataset.objects(badges__kind=NECMERGITUR).visible()
                .order_by('-metrics.followers'))
    return theme.render('nec_mergitur.html',
                        datasets=datasets,
                        badge=NECMERGITUR,
                        nb_displayed_datasets=NB_DISPLAYED_DATASETS)

 def get_context(self):
     context = super(OrganizationIssuesView, self).get_context()
     datasets = Dataset.objects(organization=self.organization)
     reuses = Reuse.objects(organization=self.organization)
     ids = [o.id for o in list(datasets) + list(reuses)]
     context['issues'] = Issue.objects(subject__in=ids)
     return context

 def check_availability(self):
     from udata.models import Dataset  # Circular imports.
     # Performances: only check the first 20 datasets for now.
     return chain(
         *[dataset.check_availability()
           for dataset in Dataset.objects(organization=self).visible()[:20]]
     )

def aggregate_datasets_daily(org, day):
    keys = ['datasets_{0}'.format(k) for k in KEYS]
    ids = [d.id for d in Dataset.objects(organization=org).only('id')]
    metrics = Metrics.objects(object_id__in=ids,
                              level='daily', date=day.isoformat())
    values = [int(metrics.sum('values.{0}'.format(k))) for k in KEYS]
    return Metrics.objects.update_daily(org, day, **dict(zip(keys, values)))

 def get_dataset(self, remote_id):
     '''Get or create a dataset given its remote ID (and its source)'''
     dataset = Dataset.objects(__raw__={
         'extras.harvest:remote_id': remote_id,
         'extras.harvest:domain': self.source.domain
     }).first()
     return dataset or Dataset()

    def test_default(self):
        org = OrganizationFactory()
        source = HarvestSourceFactory(backend='factory', organization=org)
        with self.assert_emit(signals.before_harvest_job,
                              signals.after_harvest_job):
            self.action(source.slug)

        source.reload()
        self.assertEqual(len(HarvestJob.objects(source=source)), 1)

        job = source.get_last_job()

        self.assertEqual(job.status, 'done')
        self.assertEqual(job.errors, [])
        self.assertIsNotNone(job.started)
        self.assertIsNotNone(job.ended)
        self.assertEqual(len(job.items), COUNT)

        for item in job.items:
            self.assertEqual(item.status, 'done')
            self.assertEqual(item.errors, [])
            self.assertIsNotNone(item.started)
            self.assertIsNotNone(item.ended)
            self.assertIsNotNone(item.dataset)

            dataset = item.dataset
            self.assertIsNotNone(Dataset.objects(id=dataset.id).first())
            self.assertEqual(dataset.organization, org)
            self.assertIn('harvest:remote_id', dataset.extras)
            self.assertIn('harvest:last_update', dataset.extras)
            self.assertIn('harvest:source_id', dataset.extras)

        self.assertEqual(len(HarvestJob.objects), 1)
        self.assertEqual(len(Dataset.objects), COUNT)

def openfield16():
    datasets = (Dataset.objects(badges__kind=OPENFIELD16).visible()
                .order_by('-metrics.followers'))
    return theme.render('openfield16.html',
                        datasets=datasets,
                        badge=OPENFIELD16,
                        nb_displayed_datasets=NB_DISPLAYED_DATASETS)