Python content.DataMap类代码示例

    def test_multiple(self):
        self._add([
            (1.0, 2.0, self.yesterday),
            (-10.0, 40.0, self.yesterday),
        ])
        self._queue([
            (1.0, 2.0), (1.0, 2.0),
            (40.0011, 3.0011), (40.0012, 3.0012), (40.0013, 3.0013),
            (0.0, 0.0),
            (1.0, 2.0),
            (1.00001, 2.00001),
        ])
        for shard_id in DataMap.shards():
            update_datamap.delay(batch=2, shard_id=shard_id).get()

        rows = []
        for shard in DataMap.shards().values():
            rows.extend(self.session.query(shard).all())

        self.assertEqual(len(rows), 4)
        created = set()
        modified = set()
        positions = set()
        for row in rows:
            lat, lon = row.grid
            created.add(row.created)
            modified.add(row.modified)
            positions.add((lat / 1000.0, lon / 1000.0))

        self.assertEqual(created, set([self.today, self.yesterday]))
        self.assertEqual(modified, set([self.today, self.yesterday]))
        self.assertEqual(
            positions,
            set([(1.0, 2.0), (-10.0, 40.0), (0.0, 0.0), (40.001, 3.001)]))

    def test_files(self, db_rw, session):
        today = util.utcnow().date()
        rows = [
            dict(time=today, lat=12.345, lon=12.345),
            dict(time=today, lat=0, lon=12.345),
            dict(time=today, lat=-10.000, lon=-11.000),
        ]
        for row in rows:
            lat, lon = DataMap.scale(row['lat'], row['lon'])
            data = DataMap.shard_model(lat, lon)(
                grid=(lat, lon), created=row['time'], modified=row['time'])
            session.add(data)
        session.flush()

        lines = []
        rows = 0
        db_url = str(db_rw.engine.url)
        with util.selfdestruct_tempdir() as temp_dir:
            quaddir = os.path.join(temp_dir, 'quadtrees')
            os.mkdir(quaddir)
            shapes = os.path.join(temp_dir, 'shapes')
            tiles = os.path.join(temp_dir, 'tiles')

            for shard_id, shard in DataMap.shards().items():
                filename = 'map_%s.csv.gz' % shard_id
                filepath = os.path.join(temp_dir, filename)
                result = export_file(
                    db_url, filepath, shard.__tablename__,
                    _session=session)

                if not result:
                    assert not os.path.isfile(filepath)
                    continue

                rows += result
                with util.gzip_open(filepath, 'r') as fd:
                    written = fd.read()
                lines.extend([line.split(',') for line in written.split()])

                encode_file(filename, temp_dir, quaddir, DATAMAPS_DIR)

                quadfolder = os.path.join(quaddir, 'map_' + shard_id)
                assert os.path.isdir(quadfolder)
                self._check_quadtree(quadfolder)

            merge_files(quaddir, shapes, DATAMAPS_DIR)
            self._check_quadtree(shapes)

            render_tiles(shapes, tiles, 1, 2, DATAMAPS_DIR, PNGQUANT)
            assert (sorted(os.listdir(tiles)) == ['0', '1', '2'])
            assert (sorted(os.listdir(os.path.join(tiles, '0', '0'))) ==
                    ['0.png', '[email protected]'])

        assert rows == 36
        assert len(lines) == 36
        assert (set([round(float(l[0]), 2) for l in lines]) ==
                set([-10.0, 0.0, 12.35]))
        assert (set([round(float(l[1]), 2) for l in lines]) ==
                set([-11.0, 12.35]))

    def _queue(self, pairs):
        grids = defaultdict(list)
        for lat, lon in pairs:
            lat, lon = DataMap.scale(lat, lon)
            shard_id = DataMap.shard_id(lat, lon)
            grids[shard_id].append(encode_datamap_grid(lat, lon))

        for shard_id, values in grids.items():
            queue = self.celery_app.data_queues['update_datamap_' + shard_id]
            queue.enqueue(list(values), json=False)

    def test_one(self, celery, session):
        lat = 1.234567
        lon = 2.345678
        shard_id = DataMap.shard_id(*DataMap.scale(lat, lon))
        self._queue(celery, [(lat, lon)])
        update_datamap.delay(shard_id=shard_id).get()

        grids = session.query(DataMap.shards()[shard_id]).all()
        assert len(grids) == 1
        self._check_position(grids[0], 1.235, 2.346)
        assert grids[0].created == self.today
        assert grids[0].modified == self.today

    def test_one(self):
        lat = 1.234567
        lon = 2.345678
        shard_id = DataMap.shard_id(*DataMap.scale(lat, lon))
        self._queue([(lat, lon)])
        update_datamap.delay(shard_id=shard_id).get()

        grids = self.session.query(DataMap.shards()[shard_id]).all()
        self.assertEqual(len(grids), 1)
        self._check_position(grids[0], 1.235, 2.346)
        self.assertEqual(grids[0].created, self.today)
        self.assertEqual(grids[0].modified, self.today)

    def test_files(self):
        today = util.utcnow().date()
        rows = [
            dict(time=today, lat=12.345, lon=12.345),
            dict(time=today, lat=0, lon=12.345),
            dict(time=today, lat=-10.000, lon=-11.000),
        ]
        for row in rows:
            lat, lon = DataMap.scale(row["lat"], row["lon"])
            data = DataMap.shard_model(lat, lon)(grid=(lat, lon), created=row["time"], modified=row["time"])
            self.session.add(data)
        self.session.flush()

        lines = []
        rows = 0
        with util.selfdestruct_tempdir() as temp_dir:
            quaddir = os.path.join(temp_dir, "quadtrees")
            os.mkdir(quaddir)
            shapes = os.path.join(temp_dir, "shapes")
            tiles = os.path.join(temp_dir, "tiles")

            for shard_id, shard in DATAMAP_SHARDS.items():
                filename = "map_%s.csv.gz" % shard_id
                filepath = os.path.join(temp_dir, filename)
                result = export_file(None, filepath, shard.__tablename__, _db_rw=_make_db(), _session=self.session)

                if not result:
                    self.assertFalse(os.path.isfile(filepath))
                    continue

                rows += result
                with util.gzip_open(filepath, "r") as fd:
                    written = fd.read()
                lines.extend([line.split(",") for line in written.split()])

                encode_file(filename, temp_dir, quaddir, DATAMAPS_DIR)

                quadfolder = os.path.join(quaddir, "map_" + shard_id)
                self.assertTrue(os.path.isdir(quadfolder))
                self._check_quadtree(quadfolder)

            merge_files(quaddir, shapes, DATAMAPS_DIR)
            self._check_quadtree(shapes)

            render_tiles(shapes, tiles, 1, 2, DATAMAPS_DIR, PNGQUANT)
            self.assertEqual(sorted(os.listdir(tiles)), ["0", "1", "2"])
            self.assertEqual(sorted(os.listdir(os.path.join(tiles, "0", "0"))), ["0.png", "[email protected]"])

        self.assertEqual(rows, 36)
        self.assertEqual(len(lines), 36)
        self.assertEqual(set([round(float(l[0]), 2) for l in lines]), set([-10.0, 0.0, 12.35]))
        self.assertEqual(set([round(float(l[1]), 2) for l in lines]), set([-11.0, 12.35]))

    def test_update(self):
        lat = 1.0
        lon = 2.0
        shard_id = DataMap.shard_id(*DataMap.scale(lat, lon))
        self._add([(lat, lon, self.yesterday)])
        self._queue([(lat, lon)])
        update_datamap.delay(shard_id=shard_id).get()

        grids = self.session.query(DataMap.shards()[shard_id]).all()
        self.assertEqual(len(grids), 1)
        self._check_position(grids[0], 1.0, 2.0)
        self.assertEqual(grids[0].created, self.yesterday)
        self.assertEqual(grids[0].modified, self.today)

    def test_update(self, celery, session):
        lat = 1.0
        lon = 2.0
        shard_id = DataMap.shard_id(*DataMap.scale(lat, lon))
        self._add(session, [(lat, lon, self.yesterday)])
        self._queue(celery, [(lat, lon)])
        update_datamap.delay(shard_id=shard_id).get()

        grids = session.query(DataMap.shards()[shard_id]).all()
        assert len(grids) == 1
        self._check_position(grids[0], 1.0, 2.0)
        assert grids[0].created == self.yesterday
        assert grids[0].modified == self.today

 def test_shard_id(self):
     assert DataMap.shard_id(None, None) is None
     assert DataMap.shard_id(85000, 180000) == 'ne'
     assert DataMap.shard_id(36000, 5000) == 'ne'
     assert DataMap.shard_id(35999, 5000) == 'se'
     assert DataMap.shard_id(-85000, 180000) == 'se'
     assert DataMap.shard_id(85000, -180000) == 'nw'
     assert DataMap.shard_id(36000, 4999) == 'nw'
     assert DataMap.shard_id(35999, 4999) == 'sw'
     assert DataMap.shard_id(-85000, -180000) == 'sw'

 def test_shard_id(self):
     self.assertEqual(DataMap.shard_id(None, None), None)
     self.assertEqual(DataMap.shard_id(85000, 180000), 'ne')
     self.assertEqual(DataMap.shard_id(36000, 5000), 'ne')
     self.assertEqual(DataMap.shard_id(35999, 5000), 'se')
     self.assertEqual(DataMap.shard_id(-85000, 180000), 'se')
     self.assertEqual(DataMap.shard_id(85000, -180000), 'nw')
     self.assertEqual(DataMap.shard_id(36000, 4999), 'nw')
     self.assertEqual(DataMap.shard_id(35999, 4999), 'sw')
     self.assertEqual(DataMap.shard_id(-85000, -180000), 'sw')

 def test_grid_bytes(self):
     lat = 12000
     lon = 34000
     grid = encode_datamap_grid(lat, lon)
     model = DataMap.shard_model(lat, lon)
     self.session.add(model(grid=grid))
     self.session.flush()
     result = self.session.query(model).first()
     self.assertEqual(result.grid, (lat, lon))

 def test_fields(self):
     today = util.utcnow().date()
     lat = 12345
     lon = -23456
     model = DataMap.shard_model(lat, lon)
     self.session.add(model(grid=(lat, lon), created=today, modified=today))
     self.session.flush()
     result = self.session.query(model).first()
     self.assertEqual(result.grid, (lat, lon))
     self.assertEqual(result.created, today)
     self.assertEqual(result.modified, today)

def export_files(pool, db_url, csvdir):  # pragma: no cover
    jobs = []
    result_rows = 0
    for shard_id, shard in sorted(DataMap.shards().items()):
        # sorting the shards prefers the north which contains more
        # data points than the south
        filename = os.path.join(csvdir, 'map_%s.csv.gz' % shard_id)
        jobs.append(pool.apply_async(export_file,
                                     (db_url, filename, shard.__tablename__)))

    for job in jobs:
        result_rows += job.get()

    return result_rows

    def test_multiple(self, celery, session):
        self._add(session, [
            (0.0, 1.0, self.today),
            (1.0, 2.0, self.yesterday),
            (-10.0, 40.0, self.yesterday),
        ])
        self._queue(celery, [
            (0.0, 1.0),
            (1.0, 2.0), (1.0, 2.0),
            (40.0011, 3.0011), (40.0012, 3.0012), (40.0013, 3.0013),
            (0.0, 0.0),
            (1.0, 2.0),
            (1.00001, 2.00001),
        ])
        for shard_id in DataMap.shards():
            update_datamap.delay(shard_id=shard_id).get()

        rows = []
        for shard in DataMap.shards().values():
            rows.extend(session.query(shard).all())

        assert len(rows) == 5
        created = set()
        modified = set()
        positions = set()
        for row in rows:
            lat, lon = row.grid
            created.add(row.created)
            modified.add(row.modified)
            positions.add((lat / 1000.0, lon / 1000.0))

        assert created == set([self.today, self.yesterday])
        assert modified == set([self.today, self.yesterday])
        assert (positions == set([
            (0.0, 0.0), (0.0, 1.0), (1.0, 2.0),
            (-10.0, 40.0), (40.001, 3.001)]))

    def test_cleanup(self, celery, session):
        session.add_all([
            self._one(37.0, 6.0, self.today),
            self._one(37.0, 6.1, self.today - timedelta(days=366)),
            self._one(37.0, 4.0, self.today),
            self._one(37.0, 4.1, self.today - timedelta(days=366)),
            self._one(10.0, 6.0, self.today),
            self._one(10.0, 6.1, self.today - timedelta(days=366)),
            self._one(10.0, 4.0, self.today),
            self._one(10.0, 4.1, self.today - timedelta(days=366)),
        ])
        session.flush()

        for shard_id, shard in DataMap.shards().items():
            cleanup_datamap.delay(shard_id=shard_id).get()
            assert session.query(shard).count() == 1