Python CellShard.shard_model方法代码示例

# 需要导入模块: from ichnaea.models import CellShard [as 别名]
# 或者: from ichnaea.models.CellShard import shard_model [as 别名]
    def test_update(self):
        now = util.utcnow()
        invalid_key = dict(lac=None, cid=None)
        observations = []

        def obs_factory(**kw):
            obs = CellObservationFactory.build(**kw)
            if obs is not None:
                observations.append(obs)

        cell1 = CellShardFactory(samples=3, weight=3.0)
        lat1, lon1 = (cell1.lat, cell1.lon)
        key1 = dict(radio=cell1.radio, lac=cell1.lac, cid=cell1.cid)
        obs_factory(lat=lat1, lon=lon1, created=now, **key1)
        obs_factory(lat=lat1 + 0.004, lon=lon1 + 0.006, created=now, **key1)
        obs_factory(lat=lat1 + 0.006, lon=lon1 + 0.009, created=now, **key1)
        # The lac, cid are invalid and should be skipped
        obs_factory(created=now, **invalid_key)
        obs_factory(created=now, **invalid_key)

        cell2 = CellShardFactory(
            lat=lat1 + 1.0, lon=lon1 + 1.0, samples=3, weight=3.0)
        lat2, lon2 = (cell2.lat, cell2.lon)
        key2 = dict(radio=cell2.radio, lac=cell2.lac, cid=cell2.cid)
        obs_factory(lat=lat2 + 0.001, lon=lon2 + 0.002, created=now, **key2)
        obs_factory(lat=lat2 + 0.003, lon=lon2 + 0.006, created=now, **key2)

        cell3 = CellShardFactory(samples=100000, weight=100000.0)
        lat3, lon3 = (cell3.lat, cell3.lon)
        key3 = dict(radio=cell3.radio, lac=cell3.lac, cid=cell3.cid)
        for i in range(10):
            obs_factory(lat=lat3 + 0.5, lon=lon3 + 0.5, **key3)

        self.session.commit()
        self._queue_and_update_cell(observations)

        shard = CellShard.shard_model(cell1.cellid)
        found = (self.session.query(shard)
                             .filter(shard.cellid == cell1.cellid)).one()
        self.assertAlmostEqual(found.lat, lat1 + 0.001667, 6)
        self.assertAlmostEqual(found.lon, lon1 + 0.0025, 6)

        shard = CellShard.shard_model(cell2.cellid)
        found = (self.session.query(shard)
                             .filter(shard.cellid == cell2.cellid)).one()
        self.assertAlmostEqual(found.lat, lat2 + 0.0008, 6)
        self.assertAlmostEqual(found.lon, lon2 + 0.0016, 6)

        shard = CellShard.shard_model(cell3.cellid)
        found = (self.session.query(shard)
                             .filter(shard.cellid == cell3.cellid)).one()
        expected_lat = ((lat3 * 10000) + (lat3 + 0.5) * 10) / 10010
        expected_lon = ((lon3 * 10000) + (lon3 + 0.5) * 10) / 10010
        self.assertAlmostEqual(found.lat, expected_lat, 7)
        self.assertAlmostEqual(found.lon, expected_lon, 7)

# 需要导入模块: from ichnaea.models import CellShard [as 别名]
# 或者: from ichnaea.models.CellShard import shard_model [as 别名]
def query_cells(query, lookups, model, raven_client):
    # Given a location query and a list of lookup instances, query the
    # database and return a list of model objects.
    cellids = [lookup.cellid for lookup in lookups]
    if not cellids:  # pragma: no cover
        return []

    # load all fields used in score calculation and those we
    # need for the position
    load_fields = ('lat', 'lon', 'radius', 'region', 'samples',
                   'created', 'modified', 'last_seen', 'block_last')

    today = util.utcnow().date()
    temp_blocked = today - TEMPORARY_BLOCKLIST_DURATION

    if model == CellOCID:
        # non sharded OCID table
        return query_cell_table(query.session, model, cellids,
                                temp_blocked, load_fields, raven_client)

    result = []
    shards = defaultdict(list)
    for lookup in lookups:
        shards[CellShard.shard_model(lookup.radio)].append(lookup.cellid)

    for shard, shard_cellids in shards.items():
        result.extend(
            query_cell_table(query.session, shard, shard_cellids,
                             temp_blocked, load_fields, raven_client))

    return result

# 需要导入模块: from ichnaea.models import CellShard [as 别名]
# 或者: from ichnaea.models.CellShard import shard_model [as 别名]
    def test_weighted_update(self):
        cell = CellShardFactory(radio=Radio.gsm, samples=1, weight=2.0)
        cell_lat = cell.lat
        cell_lon = cell.lon
        cell_key = dict(radio=cell.radio, mcc=cell.mcc, mnc=cell.mnc,
                        lac=cell.lac, cid=cell.cid)

        obs_factory = CellObservationFactory
        obs = [
            obs_factory(lat=cell.lat, lon=cell.lon - 0.002,
                        accuracy=20.0, signal=-51, **cell_key),
            obs_factory(lat=cell.lat, signal=-111, lon=cell.lon - 0.004,
                        accuracy=40.0, **cell_key),
        ]

        self.session.commit()
        self._queue_and_update_cell(obs)
        shard = CellShard.shard_model(cell.cellid)
        cells = self.session.query(shard).all()
        self.assertEqual(len(cells), 1)
        cell = cells[0]
        self.assertAlmostEqual(cell.lat, cell_lat)
        self.assertAlmostEqual(cell.max_lat, cell_lat)
        self.assertAlmostEqual(cell.min_lat, cell_lat)
        self.assertAlmostEqual(cell.lon, cell_lon - 0.0016358, 7)
        self.assertAlmostEqual(cell.max_lon, cell_lon)
        self.assertAlmostEqual(cell.min_lon, cell_lon - 0.004)
        self.assertEqual(cell.radius, 164)
        self.assertEqual(cell.samples, 3)
        self.assertAlmostEqual(cell.weight, 9.47, 2)

# 需要导入模块: from ichnaea.models import CellShard [as 别名]
# 或者: from ichnaea.models.CellShard import shard_model [as 别名]
    def test_max_min_radius_update(self):
        cell = CellShardFactory(radius=150, samples=3, weight=3.0)
        cell_lat = cell.lat
        cell_lon = cell.lon
        cell.max_lat = cell.lat + 0.001
        cell.min_lat = cell.lat - 0.001
        cell.max_lon = cell.lon + 0.001
        cell.min_lon = cell.lon - 0.001
        k1 = dict(radio=cell.radio, mcc=cell.mcc, mnc=cell.mnc,
                  lac=cell.lac, cid=cell.cid)

        obs_factory = CellObservationFactory
        obs = [
            obs_factory(lat=cell.lat, lon=cell.lon - 0.002, **k1),
            obs_factory(lat=cell.lat + 0.004, lon=cell.lon - 0.006, **k1),
        ]

        self.session.commit()
        self._queue_and_update_cell(obs)

        shard = CellShard.shard_model(cell.cellid)
        cells = self.session.query(shard).all()
        self.assertEqual(len(cells), 1)
        cell = cells[0]
        self.assertAlmostEqual(cell.lat, cell_lat + 0.0008)
        self.assertAlmostEqual(cell.max_lat, cell_lat + 0.004)
        self.assertAlmostEqual(cell.min_lat, cell_lat - 0.001)
        self.assertAlmostEqual(cell.lon, cell_lon - 0.0016)
        self.assertAlmostEqual(cell.max_lon, cell_lon + 0.001)
        self.assertAlmostEqual(cell.min_lon, cell_lon - 0.006)
        self.assertEqual(cell.radius, 468)
        self.assertEqual(cell.samples, 5)
        self.assertAlmostEqual(cell.weight, 5.0, 2)

# 需要导入模块: from ichnaea.models import CellShard [as 别名]
# 或者: from ichnaea.models.CellShard import shard_model [as 别名]
    def test_blocklist(self):
        now = util.utcnow()
        today = now.date()
        observations = CellObservationFactory.build_batch(3)
        obs = observations[0]
        CellShardFactory(
            radio=obs.radio, mcc=obs.mcc, mnc=obs.mnc,
            lac=obs.lac, cid=obs.cid,
            created=now,
            block_first=today - timedelta(days=10),
            block_last=today,
            block_count=1,
        )
        self.session.commit()
        self._queue_and_update_cell(observations)

        blocks = []
        for obs in observations:
            shard = CellShard.shard_model(obs.cellid)
            cell = (self.session.query(shard)
                                .filter(shard.cellid == obs.cellid)).one()
            if cell.blocked():
                blocks.append(cell)

        self.assertEqual(len(blocks), 1)
        self.check_statcounter(StatKey.cell, 2)
        self.check_statcounter(StatKey.unique_cell, 2)

# 需要导入模块: from ichnaea.models import CellShard [as 别名]
# 或者: from ichnaea.models.CellShard import shard_model [as 别名]
    def test_lock_timeout(self, celery, db_rw_drop_table,
                          redis, ro_session, session, stats):
        obs = CellObservationFactory.build()
        cell = CellShardFactory.build(
            radio=obs.radio, mcc=obs.mcc, mnc=obs.mnc,
            lac=obs.lac, cid=obs.cid,
            samples=10,
        )
        ro_session.add(cell)
        ro_session.flush()

        orig_add_area = CellUpdater.add_area_update
        orig_wait = CellUpdater._retry_wait
        num = [0]

        def mock_area(self, updated_areas, key,
                      num=num, ro_session=ro_session):
            orig_add_area(self, updated_areas, key)
            num[0] += 1
            if num[0] == 2:
                ro_session.rollback()

        try:
            CellUpdater._retry_wait = 0.0001
            session.execute('set session innodb_lock_wait_timeout = 1')
            with mock.patch.object(CellUpdater, 'add_area_update', mock_area):
                self.queue_and_update(celery, [obs])

            # the inner task logic was called exactly twice
            assert num[0] == 2

            shard = CellShard.shard_model(obs.cellid)
            cells = session.query(shard).all()
            assert len(cells) == 1
            assert cells[0].samples == 1

            self.check_statcounter(redis, StatKey.cell, 1)
            self.check_statcounter(redis, StatKey.unique_cell, 1)
            stats.check(
                counter=[('data.observation.insert', 1, ['type:cell'])],
                timer=[('task', 1, ['task:data.update_cell'])],
            )
        finally:
            CellUpdater._retry_wait = orig_wait
            for model in CellShard.shards().values():
                session.execute(text(
                    'drop table %s;' % model.__tablename__))

# 需要导入模块: from ichnaea.models import CellShard [as 别名]
# 或者: from ichnaea.models.CellShard import shard_model [as 别名]
    def test_cell_duplicated(self, celery, session):
        self.add_reports(celery, cell_factor=1, wifi_factor=0)
        # duplicate the cell entry inside the report
        item = self._pop_item(celery)
        report = item['report']
        cell = report['cellTowers'][0]
        radio = cell['radioType']
        report['cellTowers'].append(cell.copy())
        report['cellTowers'].append(cell.copy())
        report['cellTowers'][1]['signalStrength'] += 2
        report['cellTowers'][2]['signalStrength'] -= 2
        self._push_item(celery, item)
        self._update_all(session)

        shard = CellShard.shard_model(radio)
        cells = session.query(shard).all()
        assert len(cells) == 1
        assert cells[0].samples == 1

# 需要导入模块: from ichnaea.models import CellShard [as 别名]
# 或者: from ichnaea.models.CellShard import shard_model [as 别名]
    def test_cell_duplicated(self):
        self.add_reports(cell_factor=1, wifi_factor=0)
        # duplicate the cell entry inside the report
        queue = self.celery_app.export_queues['queue_export_internal']
        items = queue.dequeue(queue.queue_key())
        report = items[0]['report']
        cell = report['cellTowers'][0]
        radio = cell['radioType']
        report['cellTowers'].append(cell.copy())
        report['cellTowers'].append(cell.copy())
        report['cellTowers'][1]['signalStrength'] += 2
        report['cellTowers'][2]['signalStrength'] -= 2
        queue.enqueue(items, queue.queue_key())
        self._update_all()

        shard = CellShard.shard_model(radio)
        cells = self.session.query(shard).all()
        self.assertEqual(len(cells), 1)
        self.assertEqual(cells[0].samples, 1)

# 需要导入模块: from ichnaea.models import CellShard [as 别名]
# 或者: from ichnaea.models.CellShard import shard_model [as 别名]
    def test_lock_timeout(self):
        obs = CellObservationFactory.build()
        cell = CellShardFactory.build(
            radio=obs.radio, mcc=obs.mcc, mnc=obs.mnc,
            lac=obs.lac, cid=obs.cid,
            samples=10,
        )
        self.db_ro_session.add(cell)
        self.db_ro_session.flush()

        orig_add_area = CellUpdater.add_area_update
        orig_wait = CellUpdater._retry_wait
        num = [0]

        def mock_area(self, updated_areas, key,
                      num=num, ro_session=self.db_ro_session):
            orig_add_area(self, updated_areas, key)
            num[0] += 1
            if num[0] == 2:
                ro_session.rollback()

        try:
            CellUpdater._retry_wait = 0.001
            self.session.execute('set session innodb_lock_wait_timeout = 1')
            with mock.patch.object(CellUpdater, 'add_area_update', mock_area):
                self._queue_and_update_cell([obs])
        finally:
            CellUpdater._retry_wait = orig_wait

        # the inner task logic was called exactly twice
        self.assertEqual(num[0], 2)

        shard = CellShard.shard_model(obs.cellid)
        cells = self.session.query(shard).all()
        self.assertEqual(len(cells), 1)
        self.assertEqual(cells[0].samples, 1)

        self.check_statcounter(StatKey.cell, 1)
        self.check_statcounter(StatKey.unique_cell, 1)
        self.check_stats(
            counter=[('data.observation.insert', 1, ['type:cell'])],
            timer=[('task', 1, ['task:data.update_cell'])],
        )

# 需要导入模块: from ichnaea.models import CellShard [as 别名]
# 或者: from ichnaea.models.CellShard import shard_model [as 别名]
    def test_cell(self, celery, session):
        reports = self.add_reports(celery, cell_factor=1, wifi_factor=0)
        self._update_all(session)

        position = reports[0]['position']
        cell_data = reports[0]['cellTowers'][0]
        shard = CellShard.shard_model(cell_data['radioType'])
        cells = session.query(shard).all()
        assert len(cells) == 1
        cell = cells[0]

        assert cell.lat == position['latitude']
        assert cell.lon == position['longitude']
        assert cell.radio.name == cell_data['radioType']
        assert cell.mcc == cell_data['mobileCountryCode']
        assert cell.mnc == cell_data['mobileNetworkCode']
        assert cell.lac == cell_data['locationAreaCode']
        assert cell.cid == cell_data['cellId']
        assert cell.psc == cell_data['primaryScramblingCode']
        assert cell.samples == 1

# 需要导入模块: from ichnaea.models import CellShard [as 别名]
# 或者: from ichnaea.models.CellShard import shard_model [as 别名]
    def test_cell(self):
        reports = self.add_reports(cell_factor=1, wifi_factor=0)
        self._update_all()

        position = reports[0]['position']
        cell_data = reports[0]['cellTowers'][0]
        radio = cell_data['radioType']
        shard = CellShard.shard_model(radio)
        cells = self.session.query(shard).all()
        self.assertEqual(len(cells), 1)
        cell = cells[0]

        self.assertEqual(cell.lat, position['latitude'])
        self.assertEqual(cell.lon, position['longitude'])
        self.assertEqual(cell.radio.name, cell_data['radioType'])
        self.assertEqual(cell.mcc, cell_data['mobileCountryCode'])
        self.assertEqual(cell.mnc, cell_data['mobileNetworkCode'])
        self.assertEqual(cell.lac, cell_data['locationAreaCode'])
        self.assertEqual(cell.cid, cell_data['cellId'])
        self.assertEqual(cell.psc, cell_data['primaryScramblingCode'])
        self.assertEqual(cell.samples, 1)

# 需要导入模块: from ichnaea.models import CellShard [as 别名]
# 或者: from ichnaea.models.CellShard import shard_model [as 别名]
    def test_blocklist_moving_cells(self):
        now = util.utcnow()
        today = now.date()
        obs = []
        obs_factory = CellObservationFactory
        moving = set()
        cells = CellShardFactory.create_batch(4)
        cells.append(CellShardFactory.build())
        # a cell with an entry but no prior position
        cell = cells[0]
        cell_key = dict(radio=cell.radio, mcc=cell.mcc, mnc=cell.mnc,
                        lac=cell.lac, cid=cell.cid)
        cell.samples = None
        cell.weight = None
        obs.extend([
            obs_factory(lat=cell.lat + 0.01,
                        lon=cell.lon + 0.01, **cell_key),
            obs_factory(lat=cell.lat + 0.02,
                        lon=cell.lon + 0.05, **cell_key),
            obs_factory(lat=cell.lat + 0.03,
                        lon=cell.lon + 0.09, **cell_key),
        ])
        cell.lat = None
        cell.lon = None
        # a cell with a prior known position
        cell = cells[1]
        cell_key = dict(radio=cell.radio, mcc=cell.mcc, mnc=cell.mnc,
                        lac=cell.lac, cid=cell.cid)
        cell.samples = 1
        cell.weight = 1.0
        cell.lat += 0.1
        obs.extend([
            obs_factory(lat=cell.lat + 1.0,
                        lon=cell.lon, **cell_key),
            obs_factory(lat=cell.lat + 3.0,
                        lon=cell.lon, **cell_key),
        ])
        moving.add(cell.cellid)
        # a cell with a very different prior position
        cell = cells[2]
        cell_key = dict(radio=cell.radio, mcc=cell.mcc, mnc=cell.mnc,
                        lac=cell.lac, cid=cell.cid)
        cell.samples = 1
        cell.weight = 1.0
        obs.extend([
            obs_factory(lat=cell.lat + 3.0,
                        lon=cell.lon, **cell_key),
            obs_factory(lat=cell.lat - 0.1,
                        lon=cell.lon, **cell_key),
        ])
        moving.add(cell.cellid)
        # another cell with a prior known position (and negative lon)
        cell = cells[3]
        cell_key = dict(radio=cell.radio, mcc=cell.mcc, mnc=cell.mnc,
                        lac=cell.lac, cid=cell.cid)
        cell.samples = 1
        cell.weight = 1.0
        cell.lon *= -1.0
        obs.extend([
            obs_factory(lat=cell.lat + 1.0,
                        lon=cell.lon, **cell_key),
            obs_factory(lat=cell.lat + 2.0,
                        lon=cell.lon, **cell_key),
        ])
        moving.add(cell.cellid)
        # an already blocklisted cell
        cell = cells[4]
        cell_key = dict(radio=cell.radio, mcc=cell.mcc, mnc=cell.mnc,
                        lac=cell.lac, cid=cell.cid)
        CellShardFactory(block_first=today, block_last=today, block_count=1,
                         **cell_key)
        obs.extend([
            obs_factory(lat=cell.lat,
                        lon=cell.lon, **cell_key),
            obs_factory(lat=cell.lat + 3.0,
                        lon=cell.lon, **cell_key),
        ])
        moving.add(cell.cellid)
        self.session.commit()
        self._queue_and_update_cell(obs)

        shards = set()
        for cellid in moving:
            shards.add(CellShard.shard_model(cellid))
        blocks = []
        for shard in shards:
            for row in self.session.query(shard).all():
                if row.blocked():
                    blocks.append(row)
        self.assertEqual(set([b.cellid for b in blocks]), moving)