Python models.CellShard类代码示例

    def test_update(self):
        now = util.utcnow()
        invalid_key = dict(lac=None, cid=None)
        observations = []

        def obs_factory(**kw):
            obs = CellObservationFactory.build(**kw)
            if obs is not None:
                observations.append(obs)

        cell1 = CellShardFactory(samples=3, weight=3.0)
        lat1, lon1 = (cell1.lat, cell1.lon)
        key1 = dict(radio=cell1.radio, lac=cell1.lac, cid=cell1.cid)
        obs_factory(lat=lat1, lon=lon1, created=now, **key1)
        obs_factory(lat=lat1 + 0.004, lon=lon1 + 0.006, created=now, **key1)
        obs_factory(lat=lat1 + 0.006, lon=lon1 + 0.009, created=now, **key1)
        # The lac, cid are invalid and should be skipped
        obs_factory(created=now, **invalid_key)
        obs_factory(created=now, **invalid_key)

        cell2 = CellShardFactory(
            lat=lat1 + 1.0, lon=lon1 + 1.0, samples=3, weight=3.0)
        lat2, lon2 = (cell2.lat, cell2.lon)
        key2 = dict(radio=cell2.radio, lac=cell2.lac, cid=cell2.cid)
        obs_factory(lat=lat2 + 0.001, lon=lon2 + 0.002, created=now, **key2)
        obs_factory(lat=lat2 + 0.003, lon=lon2 + 0.006, created=now, **key2)

        cell3 = CellShardFactory(samples=100000, weight=100000.0)
        lat3, lon3 = (cell3.lat, cell3.lon)
        key3 = dict(radio=cell3.radio, lac=cell3.lac, cid=cell3.cid)
        for i in range(10):
            obs_factory(lat=lat3 + 0.5, lon=lon3 + 0.5, **key3)

        self.session.commit()
        self._queue_and_update_cell(observations)

        shard = CellShard.shard_model(cell1.cellid)
        found = (self.session.query(shard)
                             .filter(shard.cellid == cell1.cellid)).one()
        self.assertAlmostEqual(found.lat, lat1 + 0.001667, 6)
        self.assertAlmostEqual(found.lon, lon1 + 0.0025, 6)

        shard = CellShard.shard_model(cell2.cellid)
        found = (self.session.query(shard)
                             .filter(shard.cellid == cell2.cellid)).one()
        self.assertAlmostEqual(found.lat, lat2 + 0.0008, 6)
        self.assertAlmostEqual(found.lon, lon2 + 0.0016, 6)

        shard = CellShard.shard_model(cell3.cellid)
        found = (self.session.query(shard)
                             .filter(shard.cellid == cell3.cellid)).one()
        expected_lat = ((lat3 * 10000) + (lat3 + 0.5) * 10) / 10010
        expected_lon = ((lon3 * 10000) + (lon3 + 0.5) * 10) / 10010
        self.assertAlmostEqual(found.lat, expected_lat, 7)
        self.assertAlmostEqual(found.lon, expected_lon, 7)

def query_cells(query, lookups, model, raven_client):
    # Given a location query and a list of lookup instances, query the
    # database and return a list of model objects.
    cellids = [lookup.cellid for lookup in lookups]
    if not cellids:  # pragma: no cover
        return []

    # load all fields used in score calculation and those we
    # need for the position
    load_fields = ('lat', 'lon', 'radius', 'region', 'samples',
                   'created', 'modified', 'last_seen', 'block_last')

    today = util.utcnow().date()
    temp_blocked = today - TEMPORARY_BLOCKLIST_DURATION

    if model == CellOCID:
        # non sharded OCID table
        return query_cell_table(query.session, model, cellids,
                                temp_blocked, load_fields, raven_client)

    result = []
    shards = defaultdict(list)
    for lookup in lookups:
        shards[CellShard.shard_model(lookup.radio)].append(lookup.cellid)

    for shard, shard_cellids in shards.items():
        result.extend(
            query_cell_table(query.session, shard, shard_cellids,
                             temp_blocked, load_fields, raven_client))

    return result

    def test_weighted_update(self):
        cell = CellShardFactory(radio=Radio.gsm, samples=1, weight=2.0)
        cell_lat = cell.lat
        cell_lon = cell.lon
        cell_key = dict(radio=cell.radio, mcc=cell.mcc, mnc=cell.mnc,
                        lac=cell.lac, cid=cell.cid)

        obs_factory = CellObservationFactory
        obs = [
            obs_factory(lat=cell.lat, lon=cell.lon - 0.002,
                        accuracy=20.0, signal=-51, **cell_key),
            obs_factory(lat=cell.lat, signal=-111, lon=cell.lon - 0.004,
                        accuracy=40.0, **cell_key),
        ]

        self.session.commit()
        self._queue_and_update_cell(obs)
        shard = CellShard.shard_model(cell.cellid)
        cells = self.session.query(shard).all()
        self.assertEqual(len(cells), 1)
        cell = cells[0]
        self.assertAlmostEqual(cell.lat, cell_lat)
        self.assertAlmostEqual(cell.max_lat, cell_lat)
        self.assertAlmostEqual(cell.min_lat, cell_lat)
        self.assertAlmostEqual(cell.lon, cell_lon - 0.0016358, 7)
        self.assertAlmostEqual(cell.max_lon, cell_lon)
        self.assertAlmostEqual(cell.min_lon, cell_lon - 0.004)
        self.assertEqual(cell.radius, 164)
        self.assertEqual(cell.samples, 3)
        self.assertAlmostEqual(cell.weight, 9.47, 2)

    def test_max_min_radius_update(self):
        cell = CellShardFactory(radius=150, samples=3, weight=3.0)
        cell_lat = cell.lat
        cell_lon = cell.lon
        cell.max_lat = cell.lat + 0.001
        cell.min_lat = cell.lat - 0.001
        cell.max_lon = cell.lon + 0.001
        cell.min_lon = cell.lon - 0.001
        k1 = dict(radio=cell.radio, mcc=cell.mcc, mnc=cell.mnc,
                  lac=cell.lac, cid=cell.cid)

        obs_factory = CellObservationFactory
        obs = [
            obs_factory(lat=cell.lat, lon=cell.lon - 0.002, **k1),
            obs_factory(lat=cell.lat + 0.004, lon=cell.lon - 0.006, **k1),
        ]

        self.session.commit()
        self._queue_and_update_cell(obs)

        shard = CellShard.shard_model(cell.cellid)
        cells = self.session.query(shard).all()
        self.assertEqual(len(cells), 1)
        cell = cells[0]
        self.assertAlmostEqual(cell.lat, cell_lat + 0.0008)
        self.assertAlmostEqual(cell.max_lat, cell_lat + 0.004)
        self.assertAlmostEqual(cell.min_lat, cell_lat - 0.001)
        self.assertAlmostEqual(cell.lon, cell_lon - 0.0016)
        self.assertAlmostEqual(cell.max_lon, cell_lon + 0.001)
        self.assertAlmostEqual(cell.min_lon, cell_lon - 0.006)
        self.assertEqual(cell.radius, 468)
        self.assertEqual(cell.samples, 5)
        self.assertAlmostEqual(cell.weight, 5.0, 2)

    def test_blocklist(self):
        now = util.utcnow()
        today = now.date()
        observations = CellObservationFactory.build_batch(3)
        obs = observations[0]
        CellShardFactory(
            radio=obs.radio, mcc=obs.mcc, mnc=obs.mnc,
            lac=obs.lac, cid=obs.cid,
            created=now,
            block_first=today - timedelta(days=10),
            block_last=today,
            block_count=1,
        )
        self.session.commit()
        self._queue_and_update_cell(observations)

        blocks = []
        for obs in observations:
            shard = CellShard.shard_model(obs.cellid)
            cell = (self.session.query(shard)
                                .filter(shard.cellid == obs.cellid)).one()
            if cell.blocked():
                blocks.append(cell)

        self.assertEqual(len(blocks), 1)
        self.check_statcounter(StatKey.cell, 2)
        self.check_statcounter(StatKey.unique_cell, 2)

def configure_data(redis_client):
    """
    Configure fixed set of data queues.
    """
    data_queues = {
        'update_cell': DataQueue('update_cell', redis_client,
                                 queue_key='update_cell'),  # BBB
        'update_cellarea': DataQueue('update_cellarea', redis_client,
                                     queue_key='update_cellarea'),
        'update_cellarea_ocid': DataQueue('update_cellarea_ocid', redis_client,
                                          queue_key='update_cellarea_ocid'),
        'update_score': DataQueue('update_score', redis_client,
                                  queue_key='update_score'),
    }
    for shard_id in DataMap.shards().keys():
        name = 'update_datamap_' + shard_id
        data_queues[name] = DataQueue(name, redis_client, queue_key=name)
    for shard_id in CellShard.shards().keys():
        name = 'update_cell_' + shard_id
        data_queues[name] = DataQueue(
            name, redis_client, queue_key=name)
    for shard_id in WifiShard.shards().keys():
        name = 'update_wifi_' + shard_id
        data_queues[name] = DataQueue(
            name, redis_client, queue_key=name)
    return data_queues

    def test_lock_timeout(self, celery, db_rw_drop_table,
                          redis, ro_session, session, stats):
        obs = CellObservationFactory.build()
        cell = CellShardFactory.build(
            radio=obs.radio, mcc=obs.mcc, mnc=obs.mnc,
            lac=obs.lac, cid=obs.cid,
            samples=10,
        )
        ro_session.add(cell)
        ro_session.flush()

        orig_add_area = CellUpdater.add_area_update
        orig_wait = CellUpdater._retry_wait
        num = [0]

        def mock_area(self, updated_areas, key,
                      num=num, ro_session=ro_session):
            orig_add_area(self, updated_areas, key)
            num[0] += 1
            if num[0] == 2:
                ro_session.rollback()

        try:
            CellUpdater._retry_wait = 0.0001
            session.execute('set session innodb_lock_wait_timeout = 1')
            with mock.patch.object(CellUpdater, 'add_area_update', mock_area):
                self.queue_and_update(celery, [obs])

            # the inner task logic was called exactly twice
            assert num[0] == 2

            shard = CellShard.shard_model(obs.cellid)
            cells = session.query(shard).all()
            assert len(cells) == 1
            assert cells[0].samples == 1

            self.check_statcounter(redis, StatKey.cell, 1)
            self.check_statcounter(redis, StatKey.unique_cell, 1)
            stats.check(
                counter=[('data.observation.insert', 1, ['type:cell'])],
                timer=[('task', 1, ['task:data.update_cell'])],
            )
        finally:
            CellUpdater._retry_wait = orig_wait
            for model in CellShard.shards().values():
                session.execute(text(
                    'drop table %s;' % model.__tablename__))

    def _update_all(self):
        schedule_export_reports.delay().get()

        for shard_id in CellShard.shards().keys():
            update_cell.delay(shard_id=shard_id).get()

        for shard_id in WifiShard.shards().keys():
            update_wifi.delay(shard_id=shard_id).get()

    def _queue_and_update_cell(self, obs):
        sharded_obs = defaultdict(list)
        for ob in obs:
            sharded_obs[CellShard.shard_id(ob.cellid)].append(ob)

        for shard_id, values in sharded_obs.items():
            queue = self.celery_app.data_queues['update_cell_' + shard_id]
            queue.enqueue([value.to_json() for value in values])
            update_cell.delay(shard_id=shard_id).get()

    def test_import_local_cell(self):
        self.import_csv(cell_type='cell')
        cells = self.session.query(CellShard.shards()['wcdma']).all()
        self.assertEqual(len(cells), 9)

        areaids = set([cell.areaid for cell in cells])
        self.assertEqual(
            self.session.query(CellArea).count(), len(areaids))

        update_statcounter.delay(ago=0).get()
        self.check_stat(StatKey.unique_cell, 9)

    def test_import_local_cell(self, celery, redis, session):
        self.import_csv(
            celery, redis, session,
            CellShardFactory.build(radio=Radio.wcdma), cell_type='cell')
        cells = session.query(CellShard.shards()['wcdma']).all()
        assert len(cells) == 9

        areaids = set([cell.areaid for cell in cells])
        assert session.query(CellArea).count() == len(areaids)

        update_statcounter.delay().get()
        self.check_stat(session, StatKey.unique_cell, 9)

    def test_shard_queues(self):  # BBB
        observations = CellObservationFactory.build_batch(3)
        data_queues = self.celery_app.data_queues
        single_queue = data_queues['update_cell']
        single_queue.enqueue(observations)
        update_cell.delay().get()

        self.assertEqual(single_queue.size(), 0)

        total = 0
        for shard_id in CellShard.shards().keys():
            total += data_queues['update_cell_' + shard_id].size()

        self.assertEqual(total, 3)

    def _update_all(self, session, datamap_only=False):
        ExportConfigFactory(name='internal', batch=0, schema='internal')
        session.flush()
        update_incoming.delay().get()

        if datamap_only:
            return

        for shard_id in BlueShard.shards().keys():
            update_blue.delay(shard_id=shard_id).get()

        for shard_id in CellShard.shards().keys():
            update_cell.delay(shard_id=shard_id).get()

        for shard_id in WifiShard.shards().keys():
            update_wifi.delay(shard_id=shard_id).get()

    def test_cell_duplicated(self, celery, session):
        self.add_reports(celery, cell_factor=1, wifi_factor=0)
        # duplicate the cell entry inside the report
        item = self._pop_item(celery)
        report = item['report']
        cell = report['cellTowers'][0]
        radio = cell['radioType']
        report['cellTowers'].append(cell.copy())
        report['cellTowers'].append(cell.copy())
        report['cellTowers'][1]['signalStrength'] += 2
        report['cellTowers'][2]['signalStrength'] -= 2
        self._push_item(celery, item)
        self._update_all(session)

        shard = CellShard.shard_model(radio)
        cells = session.query(shard).all()
        assert len(cells) == 1
        assert cells[0].samples == 1

    def test_lock_timeout(self):
        obs = CellObservationFactory.build()
        cell = CellShardFactory.build(
            radio=obs.radio, mcc=obs.mcc, mnc=obs.mnc,
            lac=obs.lac, cid=obs.cid,
            samples=10,
        )
        self.db_ro_session.add(cell)
        self.db_ro_session.flush()

        orig_add_area = CellUpdater.add_area_update
        orig_wait = CellUpdater._retry_wait
        num = [0]

        def mock_area(self, updated_areas, key,
                      num=num, ro_session=self.db_ro_session):
            orig_add_area(self, updated_areas, key)
            num[0] += 1
            if num[0] == 2:
                ro_session.rollback()

        try:
            CellUpdater._retry_wait = 0.001
            self.session.execute('set session innodb_lock_wait_timeout = 1')
            with mock.patch.object(CellUpdater, 'add_area_update', mock_area):
                self._queue_and_update_cell([obs])
        finally:
            CellUpdater._retry_wait = orig_wait

        # the inner task logic was called exactly twice
        self.assertEqual(num[0], 2)

        shard = CellShard.shard_model(obs.cellid)
        cells = self.session.query(shard).all()
        self.assertEqual(len(cells), 1)
        self.assertEqual(cells[0].samples, 1)

        self.check_statcounter(StatKey.cell, 1)
        self.check_statcounter(StatKey.unique_cell, 1)
        self.check_stats(
            counter=[('data.observation.insert', 1, ['type:cell'])],
            timer=[('task', 1, ['task:data.update_cell'])],
        )