Python dataset.Dataset类代码示例

    def create(self):
        """
        Adds a new dataset dynamically through a POST request
        """

        # User must be authenticated so we should have a user object in
        # c.account, if not abort with error message
        if not c.account:
            abort(status_code=400, detail='user not authenticated')

        # Check if the params are there ('metadata', 'csv_file')
        if len(request.params) != 2:
            abort(status_code=400, detail='incorrect number of params')

        metadata = request.params['metadata'] \
            if 'metadata' in request.params \
            else abort(status_code=400, detail='metadata is missing')

        csv_file = request.params['csv_file'] \
            if 'csv_file' in request.params \
            else abort(status_code=400, detail='csv_file is missing')

        # We proceed with the dataset
        try:
            model = json.load(urllib2.urlopen(metadata))
        except:
            abort(status_code=400, detail='JSON model could not be parsed')
        try:
            log.info("Validating model")
            model = validate_model(model)
        except Invalid as i:
            log.error("Errors occured during model validation:")
            for field, error in i.asdict().items():
                log.error("%s: %s", field, error)
            abort(status_code=400, detail='Model is not well formed')
        dataset = Dataset.by_name(model['dataset']['name'])
        if dataset is None:
            dataset = Dataset(model)
            require.dataset.create()
            dataset.managers.append(c.account)
            dataset.private = True  # Default value
            db.session.add(dataset)
        else:
            require.dataset.update(dataset)

        log.info("Dataset: %s", dataset.name)
        source = Source(dataset=dataset, creator=c.account, url=csv_file)

        log.info(source)
        for source_ in dataset.sources:
            if source_.url == csv_file:
                source = source_
                break
        db.session.add(source)
        db.session.commit()

        # Send loading of source into celery queue
        load_source.delay(source.id)
        return to_jsonp(dataset_apply_links(dataset.as_dict()))

 def test_views_update(self):
     cra = Dataset.by_name('cra')
     views = cra.data['views']
     views[0]['label'] = 'Banana'
     response = self.app.post(url(controller='editor',
                                  action='views_update', dataset='cra'),
                              params={'views': json.dumps(views)},
                              extra_environ={'REMOTE_USER': 'test'},
                              expect_errors=True)
     assert '200' in response.status, response.status
     cra = Dataset.by_name('cra')
     assert 'Banana' in repr(cra.data['views'])

    def load_with_model_and_csv(self, metadata, csv_file, private):
        """
        Load a dataset using a metadata model file and a csv file
        """

        if metadata is None:
            response.status = 400
            return to_jsonp({'errors': 'metadata is missing'})

        if csv_file is None:
            response.status = 400
            return to_jsonp({'errors': 'csv_file is missing'})

        # We proceed with the dataset
        try:
            model = json.load(urllib2.urlopen(metadata))
        except:
            response.status = 400
            return to_jsonp({'errors': 'JSON model could not be parsed'})
        try:
            log.info("Validating model")
            model = validate_model(model)
        except Invalid as i:
            log.error("Errors occured during model validation:")
            for field, error in i.asdict().items():
                log.error("%s: %s", field, error)
            response.status = 400
            return to_jsonp({'errors': 'Model is not well formed'})
        dataset = Dataset.by_name(model['dataset']['name'])
        if dataset is None:
            dataset = Dataset(model)
            require.dataset.create()
            dataset.managers.append(c.account)
            dataset.private = private
            db.session.add(dataset)
        else:
            require.dataset.update(dataset)

        log.info("Dataset: %s", dataset.name)
        source = Source(dataset=dataset, creator=c.account,
                        url=csv_file)

        log.info(source)
        for source_ in dataset.sources:
            if source_.url == csv_file:
                source = source_
                break
        db.session.add(source)
        db.session.commit()

        # Send loading of source into celery queue
        load_source.delay(source.id)
        return to_jsonp(dataset_apply_links(dataset.as_dict()))

 def test_retract(self):
     cra = Dataset.by_name('cra')
     assert cra.private is False, cra.private
     response = self.app.post(url(controller='editor',
                                  action='retract', dataset='cra'),
                              extra_environ={'REMOTE_USER': 'test'})
     cra = Dataset.by_name('cra')
     assert cra.private is True, cra.private
     response = self.app.post(url(controller='editor',
                                  action='retract', dataset='cra'),
                              extra_environ={'REMOTE_USER': 'test'},
                              expect_errors=True)
     assert '400' in response.status, response.status

 def test_publish(self):
     cra = Dataset.by_name('cra')
     cra.private = True
     db.session.commit()
     response = self.app.post(url(controller='editor',
                                  action='publish', dataset='cra'),
                              extra_environ={'REMOTE_USER': 'test'})
     cra = Dataset.by_name('cra')
     assert cra.private is False, cra.private
     response = self.app.post(url(controller='editor',
                                  action='publish', dataset='cra'),
                              extra_environ={'REMOTE_USER': 'test'},
                              expect_errors=True)
     assert '400' in response.status, response.status

def csvimport_fixture(name):
    model_fp = csvimport_fixture_file(name, 'model.json')
    mapping_fp = csvimport_fixture_file(name, 'mapping.json')
    model = json.load(model_fp)
    if mapping_fp:
        model['mapping'] = json.load(mapping_fp)
    dataset = Dataset(model)
    dataset.generate()
    db.session.add(dataset)
    data_path = csvimport_fixture_path(name, 'data.csv')
    user = make_account()
    source = Source(dataset, user, data_path)
    db.session.add(source)
    db.session.commit()
    return source

    def test_delete(self):
        cra = Dataset.by_name('cra')
        assert len(cra) == 36, len(cra)
        # double-check authz
        response = self.app.post(url(controller='editor',
                                     action='delete', dataset='cra'),
                                 expect_errors=True)
        assert '403' in response.status
        cra = Dataset.by_name('cra')
        assert len(cra) == 36, len(cra)

        response = self.app.post(url(controller='editor',
                                     action='delete', dataset='cra'),
                                 extra_environ={'REMOTE_USER': 'test'})
        cra = Dataset.by_name('cra')
        assert cra is None, cra

    def test_new_wrong_user(self):
        # First we add a Dataset with user 'test_new'
        user = Account.by_name('test_new')
        assert user.api_key == 'd0610659-627b-4403-8b7f-6e2820ebc95d'

        u = url(controller='api/version2', action='create')
        params = {
            'metadata':
            'https://dl.dropbox.com/u/3250791/sample-openspending-model.json',
            'csv_file':
            'http://mk.ucant.org/info/data/sample-openspending-dataset.csv'
        }
        apikey_header = 'apikey {0}'.format(user.api_key)
        response = self.app.post(u, params, {'Authorization': apikey_header})

        assert "200" in response.status
        assert Dataset.by_name('openspending-example') is not None

        # After that we try to update the Dataset with user 'test_new2'
        user = Account.by_name('test_new2')
        assert user.api_key == 'c011c340-8dad-419c-8138-1c6ded86ead5'

        u = url(controller='api/version2', action='create')
        params = {
            'metadata':
            'https://dl.dropbox.com/u/3250791/sample-openspending-model.json',
            'csv_file':
            'http://mk.ucant.org/info/data/sample-openspending-dataset.csv'
        }
        apikey_header = 'apikey {0}'.format(user.api_key)
        response = self.app.post(u, params, {'Authorization': apikey_header},
                                 expect_errors=True)
        assert '403' in response.status

def archive_one(dataset_name, archive_dir):
    """
    Find the dataset, create the archive directory and start archiving
    """

    # Find the dataset
    dataset = Dataset.by_name(dataset_name)
    # If no dataset found, exit with error message
    if dataset is None:
        exit_with_error("Dataset not found. Unable to archive it.")

    # If the archive_dir exists we have to ask the user if we should overwrite
    if os.path.exists(archive_dir):
        # If user doesn't want to write over it we exit
        if not get_confirmation("%s exists. Do you want to overwrite?" % archive_dir):
            sys.exit(0)
        # If the archive dir is a file we don't do anything
        if os.path.isfile(archive_dir):
            exit_with_error("Cannot overwrite a file (need a directory).")
    # If the archive_dir doesn't exist we create it
    else:
        try:
            os.makedirs(archive_dir)
        except OSError:
            # If we couldn't create it, we exit with an error message
            exit_with_error("Couldn't create archive directory.")

    # Archive the model (dataset metadata)
    archive_model(dataset, archive_dir)
    # Archive the visualisations
    archive_visualisations(dataset, archive_dir)
    # Download all sources
    update(os.path.join(archive_dir, "sources"), dataset)

    def permissions(self):
        """
        Check a user's permissions for a given dataset. This could also be
        done via request to the user, but since we're not really doing a
        RESTful service we do this via the api instead.
        """

        # Check the parameters. Since we only use one parameter we check it
        # here instead of creating a specific parameter parser
        if len(request.params) != 1 or 'dataset' not in request.params:
            return to_jsonp({'error': 'Parameter dataset missing'})

        # Get the dataset we want to check permissions for
        dataset = Dataset.by_name(request.params['dataset'])

        # Return permissions
        return to_jsonp(
            {
                "create": can.dataset.create() and dataset is None,
                "read": False if dataset is None
                else can.dataset.read(dataset),
                "update": False if dataset is None
                else can.dataset.update(dataset),
                "delete": False if dataset is None
                else can.dataset.delete(dataset)})

 def test_index_hide_private(self):
     cra = Dataset.by_name('cra')
     cra.private = True
     db.session.commit()
     response = self.app.get(
         url(controller='dataset', action='index', format='json'))
     obj = json.loads(response.body)
     assert len(obj['datasets']) == 0

    def index(self):
        # Get all of the datasets available to the account of the logged in
        # or an anonymous user (if c.account is None)
        c.datasets = Dataset.all_by_account(c.account)
        c.territories = DatasetTerritory.dataset_counts(c.datasets)

        c.num_entries = dataset_entries(None)
        return templating.render('home/index.html')

 def test_templates_update(self):
     response = self.app.post(url(controller='editor',
                                  action='templates_update', dataset='cra'),
                              params={'serp_title': 'BANANA'},
                              extra_environ={'REMOTE_USER': 'test'},
                              expect_errors=True)
     assert '200' in response.status, response.status
     cra = Dataset.by_name('cra')
     assert cra.serp_title == 'BANANA', cra.serp_title

 def test_team_update(self):
     response = self.app.post(url(controller='editor',
                                  action='team_update', dataset='cra'),
                              params={},
                              extra_environ={'REMOTE_USER': 'test'},
                              expect_errors=True)
     assert '200' in response.status, response.status
     cra = Dataset.by_name('cra')
     assert len(cra.managers.all()) == 1, cra.managers

    def test_feeds(self):
        # Anonymous user with one public dataset
        response = self.app.get(url(controller='dataset', action='feed_rss'),
                                expect_errors=True)
        assert 'application/xml' in response.content_type
        assert '<title>Recently Created Datasets</title>' in response
        assert '<item><title>Country Regional Analysis v2009' in response
        cra = Dataset.by_name('cra')
        cra.private = True
        db.session.add(cra)
        db.session.commit()

        # Anonymous user with one private dataset
        response = self.app.get(url(controller='dataset', action='feed_rss'),
                                expect_errors=True)
        assert 'application/xml' in response.content_type
        assert '<title>Recently Created Datasets</title>' in response
        assert '<item><title>Country Regional Analysis v2009' not in response

        # Logged in user with one public dataset
        cra.private = False
        db.session.add(cra)
        db.session.commit()
        response = self.app.get(url(controller='dataset', action='feed_rss'),
                                expect_errors=True,
                                extra_environ={'REMOTE_USER': 'test'})
        assert 'application/xml' in response.content_type
        assert '<title>Recently Created Datasets</title>' in response
        assert '<item><title>Country Regional Analysis v2009' in response

        # Logged in user with one private dataset
        cra.private = True
        db.session.add(cra)
        db.session.commit()
        response = self.app.get(url(controller='dataset', action='feed_rss'),
                                expect_errors=True,
                                extra_environ={'REMOTE_USER': 'test'})
        assert 'application/xml' in response.content_type
        assert '<title>Recently Created Datasets</title>' in response
        assert '<item><title>Country Regional Analysis v2009' not in response

        # Logged in admin user with one private dataset
        admin_user = make_account('admin')
        admin_user.admin = True
        db.session.add(admin_user)
        db.session.commit()
        response = self.app.get(url(controller='dataset', action='feed_rss'),
                                extra_environ={'REMOTE_USER': 'admin'})
        assert '<title>Recently Created Datasets</title>' in response
        assert '<item><title>Country Regional Analysis v2009' in response
        assert 'application/xml' in response.content_type

        response = self.app.get(url(controller='dataset', action='index'))
        assert ('<link rel="alternate" type="application/rss+xml" title="'
                'Latest Datasets on OpenSpending" href="/datasets.rss"' in
                response)