Python models.ShapeMetadata类代码示例

def export_shape(dataset_name):
    """Route for /shapes/<shapeset>/ endpoint. Requires a dataset argument
    and can apply column specific filters to it.

    :param dataset_name: user provided name of target shapeset
    :returns: response object result of _export_shape
    """
    # Find a way to work these into the validator, they shouldn't be out here.
    if dataset_name not in ShapeMetadata.tablenames():
        return make_error(dataset_name + ' not found.', 404)
    try:
        ShapeMetadata.get_by_dataset_name(dataset_name).shape_table
    except NoSuchTableError:
        return make_error(dataset_name + ' has yet to be ingested.', 404)

    meta_params = ('shape', 'data_type', 'location_geom__within', 'job')
    request_args = request.args.to_dict()

    # Using the 'shape' key triggers the correct validator.
    request_args['shape'] = dataset_name
    validated_args = validate(
        ExportFormatsValidator(only=meta_params),
        request_args
    )

    if validated_args.errors:
        return bad_request(validated_args.errors)
    elif validated_args.data.get('job'):
        return make_job_response('export-shape', validated_args)
    else:
        query = _export_shape(validated_args)
        shapeset = validated_args.data.get('shapeset')
        data_type = validated_args.data.get('data_type')
        return export_dataset_to_response(shapeset, data_type, query)

    def test_delete_shape(self):
        # Can we remove a shape that's fully ingested?
        city_meta = postgres_session.query(ShapeMetadata).get(shape_fixtures['city'].table_name)
        self.assertIsNotNone(city_meta)
        city_meta.remove_table()
        postgres_session.commit()
        city_meta = postgres_session.query(ShapeMetadata).get(shape_fixtures['city'].table_name)
        self.assertIsNone(city_meta)

        # Can we remove a shape that's only in the metadata?
        dummy_meta = postgres_session.query(ShapeMetadata).get(self.dummy_name)
        self.assertIsNotNone(dummy_meta)
        dummy_meta.remove_table()
        postgres_session.commit()
        dummy_meta = postgres_session.query(ShapeMetadata).get(self.dummy_name)
        self.assertIsNone(dummy_meta)

        # Add them back to return to original test state
        ShapeTests.ingest_fixture(shape_fixtures['city'])
        ShapeMetadata.add(human_name='Dummy Name',
                          source_url=None,
                          update_freq='yearly',
                          approved_status=False)

        postgres_session.commit()

def get_all_shape_datasets():
    """
    Fetches metadata for every shape dataset in meta_shape
    """
    try:
        response_skeleton = {
                'meta': {
                    'status': 'ok',
                    'message': '',
                },
                'objects': []
            }

        geom = request.args.get('location_geom__within')
        if geom:
            geom = make_sql_ready_geom(geom)

        public_listing = ShapeMetadata.index(geom)
        response_skeleton['objects'] = public_listing
        status_code = 200

    except Exception as e:
        print e.message
        response_skeleton = {
            'meta': {
                'status': 'error',
                'message': '',
            }
        }
        status_code = 500

    resp = make_response(json.dumps(response_skeleton), status_code)
    resp.headers['Content-Type'] = 'application/json'
    return resp

def get_all_shape_datasets():
    """
    Fetches metadata for every shape dataset in meta_shape
    """
    try:
        response_skeleton = {
                'meta': {
                    'status': 'ok',
                    'message': '',
                },
                'objects': []
            }

        public_listing = ShapeMetadata.index(caller_session=session)
        response_skeleton['objects'] = public_listing
        status_code = 200

    except Exception as e:
        print e.message
        response_skeleton = {
            'meta': {
                'status': 'error',
                'message': '',
            }
        }
        status_code = 500

    resp = make_response(json.dumps(response_skeleton), status_code)
    resp.headers['Content-Type'] = 'application/json'
    return resp

 def ingest_fixture(fixture):
     # Add the fixture to the metadata first
     shape_meta = ShapeMetadata.add(caller_session=session, human_name=fixture.human_name, source_url=None)
     session.commit()
     # Bypass the celery task and call on a ShapeETL directly
     ShapeETL(meta=shape_meta, source_path=fixture.path).import_shapefile()
     return shape_meta

def _export_dataset_to_response(shapeset, data_type, query=None):
    export_format = unicode.lower(unicode(data_type))

    # Make a filename that we are reasonably sure to be unique and not occupied by anyone else.
    sacrifice_file = tempfile.NamedTemporaryFile()
    export_path = sacrifice_file.name
    sacrifice_file.close()  # Removes file from system.

    try:
        # Write to that filename.
        OgrExport(export_format, export_path, shapeset.name, query).write_file()
        # Dump it in the response.
        with open(export_path, 'r') as to_export:
            resp = make_response(to_export.read(), 200)

        extension = _shape_format_to_file_extension(export_format)

        # Make the downloaded filename look nice
        shapemeta = ShapeMetadata.get_by_dataset_name(shapeset.name)
        resp.headers['Content-Type'] = _shape_format_to_content_header(export_format)
        resp.headers['Content-Disposition'] = 'attachment; filename={}.{}'.format(shapemeta.human_name, extension)
        return resp

    except Exception as e:
        error_message = 'Failed to export shape dataset {}'.format(shapeset.name)
        print repr(e)
        return make_response(error_message, 500)
    finally:
        # Don't leave that file hanging around.
        if os.path.isfile(export_path):
            os.remove(export_path)

 def ingest_fixture(fixture):
     # Add the fixture to the metadata first
     shape_meta = ShapeMetadata.add(human_name=fixture.human_name,
                                    source_url=None,
                                    update_freq=fixture.update_freq,
                                    approved_status=False)
     session.commit()
     # Bypass the celery task and call on a ShapeETL directly
     ShapeETL(meta=shape_meta, source_path=fixture.path).add()
     return shape_meta

def view_datasets():
    datasets_pending = fetch_pending_tables(MetaTable)
    shapes_pending = fetch_pending_tables(ShapeMetadata)
    datasets = MetaTable.get_all_with_etl_status()
    shapesets = ShapeMetadata.get_all_with_etl_status()

    return render_template('admin/view-datasets.html',
                           datasets_pending=datasets_pending,
                           shapes_pending=shapes_pending,
                           datasets=datasets,
                           shape_datasets=shapesets)

    def test_delete_shape(self):
        # Can we remove a shape that's fully ingested?
        city_meta = session.query(ShapeMetadata).get(fixtures["city"].table_name)
        self.assertIsNotNone(city_meta)
        city_meta.remove_table(caller_session=session)
        session.commit()
        city_meta = session.query(ShapeMetadata).get(fixtures["city"].table_name)
        self.assertIsNone(city_meta)

        # Can we remove a shape that's only in the metadata?
        dummy_meta = session.query(ShapeMetadata).get(self.dummy_name)
        self.assertIsNotNone(dummy_meta)
        dummy_meta.remove_table(caller_session=session)
        session.commit()
        dummy_meta = session.query(ShapeMetadata).get(self.dummy_name)
        self.assertIsNone(dummy_meta)

        # Add them back to return to original test state
        ShapeTests.ingest_fixture(fixtures["city"])
        ShapeMetadata.add(caller_session=session, human_name=u"Dummy Name", source_url=None)
        session.commit()

def get_all_shape_datasets():
    """Fetches metadata for every shape dataset in meta_shape.
    """
    try:
        response_skeleton = {
            'meta': {
                'status': 'ok',
                'message': '',
            },
            'objects': []
        }

        geom = request.args.get('location_geom__within')
        simple_bbox = request.args.get('simple_bbox')

        if geom:
            geom = make_fragment_str(
                extract_first_geometry_fragment(geom)
            )

        if simple_bbox:
            public_listing = ShapeMetadata.simple_index(geom)
        else:
            public_listing = ShapeMetadata.index(geom)
        response_skeleton['objects'] = public_listing
        status_code = 200

    except Exception as e:
        response_skeleton = {
            'meta': {
                'status': 'error',
                'message': str(e),
            }
        }
        status_code = 500

    resp = make_response(json.dumps(response_skeleton, default=str), status_code)
    resp.headers['Content-Type'] = 'application/json'
    return resp

def shape_meta_from_submit_form(form, is_approved):
    md = ShapeMetadata.add(
        human_name=form['dataset_name'],
        source_url=form['file_url'],
        view_url=form.get('view_url'),
        attribution=form.get('dataset_attribution'),
        description=form.get('dataset_description'),
        update_freq=form['update_frequency'],
        contributor_name=form['contributor_name'],
        contributor_organization=form.get('contributor_organization'),
        contributor_email=form['contributor_email'],
        approved_status=is_approved)
    postgres_session.commit()
    return md

    def ingest_shapes(cls):
        fixtures = [f for k, f in shape_fixtures.items() if k != 'changed_neighborhoods']
        fixture_table_names = [f.table_name for f in fixtures]
        drop_tables(fixture_table_names)
        postgres_session.commit()

        for fixture in fixtures:
            cls.ingest_fixture(fixture)

        # Add a dummy dataset to the metadata without ingesting a shapefile for it
        cls.dummy_name = ShapeMetadata.add(human_name='Dummy Name',
                                           source_url=None,
                                           update_freq='yearly',
                                           approved_status=False).dataset_name
        postgres_session.commit()

    def setUpClass(cls, shutdown=False):

        # Remove tables that we're about to recreate.
        # This doesn't happen in teardown because I find it helpful
        # to inspect them in the DB after running the tests.
        meta_table_names = ['meta_shape']
        fixture_table_names = [fixture.table_name for key, fixture in fixtures.iteritems()]

        drop_tables(meta_table_names + fixture_table_names)

        # Re-add meta tables
        init_meta()

        # Fully ingest the fixtures
        BasePlenarioTest.ingest_fixture(fixtures['city'])
        BasePlenarioTest.ingest_fixture(fixtures['streets'])
        BasePlenarioTest.ingest_fixture(fixtures['zips'])
        BasePlenarioTest.ingest_fixture(fixtures['neighborhoods'])

        # Add a dummy dataset to the metadata without ingesting a shapefile for it
        cls.dummy_name = ShapeMetadata.add(human_name=u'Dummy Name',
                                           source_url=None,
                                           update_freq='yearly',
                                           approved_status=False).dataset_name
        session.commit()

        tables_to_drop = [
            'flu_shot_clinics',
            'landmarks',
            'crimes',
            'meta_master'
        ]
        drop_tables(tables_to_drop)

        init_meta()

        ingest_from_fixture(flu_shot_meta, flu_path)
        ingest_from_fixture(landmarks_meta, landmarks_path)
        ingest_from_fixture(crime_meta, crime_path)

        cls.app = create_app().test_client()

        '''/detail'''

def view_datasets():
    datasets_pending = session.query(MetaTable).\
        filter(MetaTable.approved_status != True).\
        all()

    shapes_pending = session.query(ShapeMetadata).\
        filter(ShapeMetadata.approved_status != True).\
        all()

    try:
        q = text(''' 
            SELECT m.*, c.status, c.task_id
            FROM meta_master AS m 
            LEFT JOIN celery_taskmeta AS c 
              ON c.id = (
                SELECT id FROM celery_taskmeta 
                WHERE task_id = ANY(m.result_ids) 
                ORDER BY date_done DESC 
                LIMIT 1
              )
            WHERE m.approved_status = 'true'
        ''')
        with engine.begin() as c:
            datasets = list(c.execute(q))
    except NoSuchTableError:
        datasets = session.query(MetaTable)\
            .filter(MetaTable.approved_status == True)\
            .all()

    try:
        shape_datasets = ShapeMetadata.get_all_with_etl_status()
    except NoSuchTableError:
        # If we can't find shape metadata, soldier on.
        shape_datasets = None

    return render_template('admin/view-datasets.html',
                           datasets_pending=datasets_pending,
                           shapes_pending=shapes_pending,
                           datasets=datasets,
                           shape_datasets=shape_datasets)

    def setUpClass(cls):

        # Remove tables that we're about to recreate.
        # This doesn't happen in teardown because I find it helpful to inspect them in the DB after running the tests.
        meta_table_names = ["dat_master", "meta_shape", "meta_master", "plenario_user"]
        fixture_table_names = [fixture.table_name for key, fixture in fixtures.iteritems()]
        drop_tables(meta_table_names + fixture_table_names)

        # Re-add meta tables
        init_master_meta_user()

        # Fully ingest the fixtures
        ShapeTests.ingest_fixture(fixtures["city"])
        ShapeTests.ingest_fixture(fixtures["streets"])
        ShapeTests.ingest_fixture(fixtures["zips"])

        # Add a dummy dataset to the metadata without ingesting a shapefile for it
        cls.dummy_name = ShapeMetadata.add(
            caller_session=session, human_name=u"Dummy Name", source_url=None
        ).dataset_name
        session.commit()

        cls.app = create_app().test_client()