Python DataMap.shards方法代码示例

# 需要导入模块: from ichnaea.models.content import DataMap [as 别名]
# 或者: from ichnaea.models.content.DataMap import shards [as 别名]
    def test_multiple(self):
        self._add([
            (1.0, 2.0, self.yesterday),
            (-10.0, 40.0, self.yesterday),
        ])
        self._queue([
            (1.0, 2.0), (1.0, 2.0),
            (40.0011, 3.0011), (40.0012, 3.0012), (40.0013, 3.0013),
            (0.0, 0.0),
            (1.0, 2.0),
            (1.00001, 2.00001),
        ])
        for shard_id in DataMap.shards():
            update_datamap.delay(batch=2, shard_id=shard_id).get()

        rows = []
        for shard in DataMap.shards().values():
            rows.extend(self.session.query(shard).all())

        self.assertEqual(len(rows), 4)
        created = set()
        modified = set()
        positions = set()
        for row in rows:
            lat, lon = row.grid
            created.add(row.created)
            modified.add(row.modified)
            positions.add((lat / 1000.0, lon / 1000.0))

        self.assertEqual(created, set([self.today, self.yesterday]))
        self.assertEqual(modified, set([self.today, self.yesterday]))
        self.assertEqual(
            positions,
            set([(1.0, 2.0), (-10.0, 40.0), (0.0, 0.0), (40.001, 3.001)]))

# 需要导入模块: from ichnaea.models.content import DataMap [as 别名]
# 或者: from ichnaea.models.content.DataMap import shards [as 别名]
    def test_files(self, db_rw, session):
        today = util.utcnow().date()
        rows = [
            dict(time=today, lat=12.345, lon=12.345),
            dict(time=today, lat=0, lon=12.345),
            dict(time=today, lat=-10.000, lon=-11.000),
        ]
        for row in rows:
            lat, lon = DataMap.scale(row['lat'], row['lon'])
            data = DataMap.shard_model(lat, lon)(
                grid=(lat, lon), created=row['time'], modified=row['time'])
            session.add(data)
        session.flush()

        lines = []
        rows = 0
        db_url = str(db_rw.engine.url)
        with util.selfdestruct_tempdir() as temp_dir:
            quaddir = os.path.join(temp_dir, 'quadtrees')
            os.mkdir(quaddir)
            shapes = os.path.join(temp_dir, 'shapes')
            tiles = os.path.join(temp_dir, 'tiles')

            for shard_id, shard in DataMap.shards().items():
                filename = 'map_%s.csv.gz' % shard_id
                filepath = os.path.join(temp_dir, filename)
                result = export_file(
                    db_url, filepath, shard.__tablename__,
                    _session=session)

                if not result:
                    assert not os.path.isfile(filepath)
                    continue

                rows += result
                with util.gzip_open(filepath, 'r') as fd:
                    written = fd.read()
                lines.extend([line.split(',') for line in written.split()])

                encode_file(filename, temp_dir, quaddir, DATAMAPS_DIR)

                quadfolder = os.path.join(quaddir, 'map_' + shard_id)
                assert os.path.isdir(quadfolder)
                self._check_quadtree(quadfolder)

            merge_files(quaddir, shapes, DATAMAPS_DIR)
            self._check_quadtree(shapes)

            render_tiles(shapes, tiles, 1, 2, DATAMAPS_DIR, PNGQUANT)
            assert (sorted(os.listdir(tiles)) == ['0', '1', '2'])
            assert (sorted(os.listdir(os.path.join(tiles, '0', '0'))) ==
                    ['0.png', '[email protected]'])

        assert rows == 36
        assert len(lines) == 36
        assert (set([round(float(l[0]), 2) for l in lines]) ==
                set([-10.0, 0.0, 12.35]))
        assert (set([round(float(l[1]), 2) for l in lines]) ==
                set([-11.0, 12.35]))

# 需要导入模块: from ichnaea.models.content import DataMap [as 别名]
# 或者: from ichnaea.models.content.DataMap import shards [as 别名]
    def test_one(self, celery, session):
        lat = 1.234567
        lon = 2.345678
        shard_id = DataMap.shard_id(*DataMap.scale(lat, lon))
        self._queue(celery, [(lat, lon)])
        update_datamap.delay(shard_id=shard_id).get()

        grids = session.query(DataMap.shards()[shard_id]).all()
        assert len(grids) == 1
        self._check_position(grids[0], 1.235, 2.346)
        assert grids[0].created == self.today
        assert grids[0].modified == self.today

# 需要导入模块: from ichnaea.models.content import DataMap [as 别名]
# 或者: from ichnaea.models.content.DataMap import shards [as 别名]
    def test_one(self):
        lat = 1.234567
        lon = 2.345678
        shard_id = DataMap.shard_id(*DataMap.scale(lat, lon))
        self._queue([(lat, lon)])
        update_datamap.delay(shard_id=shard_id).get()

        grids = self.session.query(DataMap.shards()[shard_id]).all()
        self.assertEqual(len(grids), 1)
        self._check_position(grids[0], 1.235, 2.346)
        self.assertEqual(grids[0].created, self.today)
        self.assertEqual(grids[0].modified, self.today)

# 需要导入模块: from ichnaea.models.content import DataMap [as 别名]
# 或者: from ichnaea.models.content.DataMap import shards [as 别名]
    def test_update(self):
        lat = 1.0
        lon = 2.0
        shard_id = DataMap.shard_id(*DataMap.scale(lat, lon))
        self._add([(lat, lon, self.yesterday)])
        self._queue([(lat, lon)])
        update_datamap.delay(shard_id=shard_id).get()

        grids = self.session.query(DataMap.shards()[shard_id]).all()
        self.assertEqual(len(grids), 1)
        self._check_position(grids[0], 1.0, 2.0)
        self.assertEqual(grids[0].created, self.yesterday)
        self.assertEqual(grids[0].modified, self.today)

# 需要导入模块: from ichnaea.models.content import DataMap [as 别名]
# 或者: from ichnaea.models.content.DataMap import shards [as 别名]
    def test_update(self, celery, session):
        lat = 1.0
        lon = 2.0
        shard_id = DataMap.shard_id(*DataMap.scale(lat, lon))
        self._add(session, [(lat, lon, self.yesterday)])
        self._queue(celery, [(lat, lon)])
        update_datamap.delay(shard_id=shard_id).get()

        grids = session.query(DataMap.shards()[shard_id]).all()
        assert len(grids) == 1
        self._check_position(grids[0], 1.0, 2.0)
        assert grids[0].created == self.yesterday
        assert grids[0].modified == self.today

# 需要导入模块: from ichnaea.models.content import DataMap [as 别名]
# 或者: from ichnaea.models.content.DataMap import shards [as 别名]
def export_files(pool, db_url, csvdir):  # pragma: no cover
    jobs = []
    result_rows = 0
    for shard_id, shard in sorted(DataMap.shards().items()):
        # sorting the shards prefers the north which contains more
        # data points than the south
        filename = os.path.join(csvdir, 'map_%s.csv.gz' % shard_id)
        jobs.append(pool.apply_async(export_file,
                                     (db_url, filename, shard.__tablename__)))

    for job in jobs:
        result_rows += job.get()

    return result_rows

# 需要导入模块: from ichnaea.models.content import DataMap [as 别名]
# 或者: from ichnaea.models.content.DataMap import shards [as 别名]
    def test_multiple(self, celery, session):
        self._add(session, [
            (0.0, 1.0, self.today),
            (1.0, 2.0, self.yesterday),
            (-10.0, 40.0, self.yesterday),
        ])
        self._queue(celery, [
            (0.0, 1.0),
            (1.0, 2.0), (1.0, 2.0),
            (40.0011, 3.0011), (40.0012, 3.0012), (40.0013, 3.0013),
            (0.0, 0.0),
            (1.0, 2.0),
            (1.00001, 2.00001),
        ])
        for shard_id in DataMap.shards():
            update_datamap.delay(shard_id=shard_id).get()

        rows = []
        for shard in DataMap.shards().values():
            rows.extend(session.query(shard).all())

        assert len(rows) == 5
        created = set()
        modified = set()
        positions = set()
        for row in rows:
            lat, lon = row.grid
            created.add(row.created)
            modified.add(row.modified)
            positions.add((lat / 1000.0, lon / 1000.0))

        assert created == set([self.today, self.yesterday])
        assert modified == set([self.today, self.yesterday])
        assert (positions == set([
            (0.0, 0.0), (0.0, 1.0), (1.0, 2.0),
            (-10.0, 40.0), (40.001, 3.001)]))

# 需要导入模块: from ichnaea.models.content import DataMap [as 别名]
# 或者: from ichnaea.models.content.DataMap import shards [as 别名]
    def test_cleanup(self, celery, session):
        session.add_all([
            self._one(37.0, 6.0, self.today),
            self._one(37.0, 6.1, self.today - timedelta(days=366)),
            self._one(37.0, 4.0, self.today),
            self._one(37.0, 4.1, self.today - timedelta(days=366)),
            self._one(10.0, 6.0, self.today),
            self._one(10.0, 6.1, self.today - timedelta(days=366)),
            self._one(10.0, 4.0, self.today),
            self._one(10.0, 4.1, self.today - timedelta(days=366)),
        ])
        session.flush()

        for shard_id, shard in DataMap.shards().items():
            cleanup_datamap.delay(shard_id=shard_id).get()
            assert session.query(shard).count() == 1

# 需要导入模块: from ichnaea.models.content import DataMap [as 别名]
# 或者: from ichnaea.models.content.DataMap import shards [as 别名]
 def __init__(self, task, session, pipe, shard_id=None):
     DataTask.__init__(self, task, session)
     self.pipe = pipe
     self.shard_id = shard_id
     self.shard = DataMap.shards().get(shard_id)

# 需要导入模块: from ichnaea.models.content import DataMap [as 别名]
# 或者: from ichnaea.models.content.DataMap import shards [as 别名]
 def __init__(self, task, shard_id=None):
     self.task = task
     self.shard_id = shard_id
     self.shard = DataMap.shards().get(shard_id)

# 需要导入模块: from ichnaea.models.content import DataMap [as 别名]
# 或者: from ichnaea.models.content.DataMap import shards [as 别名]
 def test_empty(self):
     for shard_id, shard in DataMap.shards().items():
         update_datamap.delay(shard_id=shard_id).get()
         self.assertEqual(self.session.query(shard).count(), 0)

# 需要导入模块: from ichnaea.models.content import DataMap [as 别名]
# 或者: from ichnaea.models.content.DataMap import shards [as 别名]
 def test_empty(self, celery, session):
     for shard_id, shard in DataMap.shards().items():
         update_datamap.delay(shard_id=shard_id).get()
         assert session.query(shard).count() == 0