Python CellShard.shards方法代码示例

# 需要导入模块: from ichnaea.models import CellShard [as 别名]
# 或者: from ichnaea.models.CellShard import shards [as 别名]
def configure_data(redis_client):
    """
    Configure fixed set of data queues.
    """
    data_queues = {
        'update_cell': DataQueue('update_cell', redis_client,
                                 queue_key='update_cell'),  # BBB
        'update_cellarea': DataQueue('update_cellarea', redis_client,
                                     queue_key='update_cellarea'),
        'update_cellarea_ocid': DataQueue('update_cellarea_ocid', redis_client,
                                          queue_key='update_cellarea_ocid'),
        'update_score': DataQueue('update_score', redis_client,
                                  queue_key='update_score'),
    }
    for shard_id in DataMap.shards().keys():
        name = 'update_datamap_' + shard_id
        data_queues[name] = DataQueue(name, redis_client, queue_key=name)
    for shard_id in CellShard.shards().keys():
        name = 'update_cell_' + shard_id
        data_queues[name] = DataQueue(
            name, redis_client, queue_key=name)
    for shard_id in WifiShard.shards().keys():
        name = 'update_wifi_' + shard_id
        data_queues[name] = DataQueue(
            name, redis_client, queue_key=name)
    return data_queues

# 需要导入模块: from ichnaea.models import CellShard [as 别名]
# 或者: from ichnaea.models.CellShard import shards [as 别名]
    def _update_all(self):
        schedule_export_reports.delay().get()

        for shard_id in CellShard.shards().keys():
            update_cell.delay(shard_id=shard_id).get()

        for shard_id in WifiShard.shards().keys():
            update_wifi.delay(shard_id=shard_id).get()

# 需要导入模块: from ichnaea.models import CellShard [as 别名]
# 或者: from ichnaea.models.CellShard import shards [as 别名]
    def test_import_local_cell(self):
        self.import_csv(cell_type='cell')
        cells = self.session.query(CellShard.shards()['wcdma']).all()
        self.assertEqual(len(cells), 9)

        areaids = set([cell.areaid for cell in cells])
        self.assertEqual(
            self.session.query(CellArea).count(), len(areaids))

        update_statcounter.delay(ago=0).get()
        self.check_stat(StatKey.unique_cell, 9)

# 需要导入模块: from ichnaea.models import CellShard [as 别名]
# 或者: from ichnaea.models.CellShard import shards [as 别名]
    def test_import_local_cell(self, celery, redis, session):
        self.import_csv(
            celery, redis, session,
            CellShardFactory.build(radio=Radio.wcdma), cell_type='cell')
        cells = session.query(CellShard.shards()['wcdma']).all()
        assert len(cells) == 9

        areaids = set([cell.areaid for cell in cells])
        assert session.query(CellArea).count() == len(areaids)

        update_statcounter.delay().get()
        self.check_stat(session, StatKey.unique_cell, 9)

# 需要导入模块: from ichnaea.models import CellShard [as 别名]
# 或者: from ichnaea.models.CellShard import shards [as 别名]
    def test_shard_queues(self):  # BBB
        observations = CellObservationFactory.build_batch(3)
        data_queues = self.celery_app.data_queues
        single_queue = data_queues['update_cell']
        single_queue.enqueue(observations)
        update_cell.delay().get()

        self.assertEqual(single_queue.size(), 0)

        total = 0
        for shard_id in CellShard.shards().keys():
            total += data_queues['update_cell_' + shard_id].size()

        self.assertEqual(total, 3)

# 需要导入模块: from ichnaea.models import CellShard [as 别名]
# 或者: from ichnaea.models.CellShard import shards [as 别名]
    def _update_all(self, session, datamap_only=False):
        ExportConfigFactory(name='internal', batch=0, schema='internal')
        session.flush()
        update_incoming.delay().get()

        if datamap_only:
            return

        for shard_id in BlueShard.shards().keys():
            update_blue.delay(shard_id=shard_id).get()

        for shard_id in CellShard.shards().keys():
            update_cell.delay(shard_id=shard_id).get()

        for shard_id in WifiShard.shards().keys():
            update_wifi.delay(shard_id=shard_id).get()

# 需要导入模块: from ichnaea.models import CellShard [as 别名]
# 或者: from ichnaea.models.CellShard import shards [as 别名]
    def test_lock_timeout(self, celery, db_rw_drop_table,
                          redis, ro_session, session, stats):
        obs = CellObservationFactory.build()
        cell = CellShardFactory.build(
            radio=obs.radio, mcc=obs.mcc, mnc=obs.mnc,
            lac=obs.lac, cid=obs.cid,
            samples=10,
        )
        ro_session.add(cell)
        ro_session.flush()

        orig_add_area = CellUpdater.add_area_update
        orig_wait = CellUpdater._retry_wait
        num = [0]

        def mock_area(self, updated_areas, key,
                      num=num, ro_session=ro_session):
            orig_add_area(self, updated_areas, key)
            num[0] += 1
            if num[0] == 2:
                ro_session.rollback()

        try:
            CellUpdater._retry_wait = 0.0001
            session.execute('set session innodb_lock_wait_timeout = 1')
            with mock.patch.object(CellUpdater, 'add_area_update', mock_area):
                self.queue_and_update(celery, [obs])

            # the inner task logic was called exactly twice
            assert num[0] == 2

            shard = CellShard.shard_model(obs.cellid)
            cells = session.query(shard).all()
            assert len(cells) == 1
            assert cells[0].samples == 1

            self.check_statcounter(redis, StatKey.cell, 1)
            self.check_statcounter(redis, StatKey.unique_cell, 1)
            stats.check(
                counter=[('data.observation.insert', 1, ['type:cell'])],
                timer=[('task', 1, ['task:data.update_cell'])],
            )
        finally:
            CellUpdater._retry_wait = orig_wait
            for model in CellShard.shards().values():
                session.execute(text(
                    'drop table %s;' % model.__tablename__))

# 需要导入模块: from ichnaea.models import CellShard [as 别名]
# 或者: from ichnaea.models.CellShard import shards [as 别名]
def configure_data(redis_client):
    """
    Configure fixed set of data queues.
    """
    data_queues = {
        # update_incoming needs to be the exact same as in webapp.config
        'update_incoming': DataQueue('update_incoming', redis_client,
                                     batch=100, compress=True),
    }
    for key in ('update_cellarea', 'update_cellarea_ocid'):
        data_queues[key] = DataQueue(key, redis_client, batch=100, json=False)
    for shard_id in BlueShard.shards().keys():
        key = 'update_blue_' + shard_id
        data_queues[key] = DataQueue(key, redis_client, batch=500)
    for shard_id in DataMap.shards().keys():
        key = 'update_datamap_' + shard_id
        data_queues[key] = DataQueue(key, redis_client, batch=500, json=False)
    for shard_id in CellShard.shards().keys():
        key = 'update_cell_' + shard_id
        data_queues[key] = DataQueue(key, redis_client, batch=500)
    for shard_id in WifiShard.shards().keys():
        key = 'update_wifi_' + shard_id
        data_queues[key] = DataQueue(key, redis_client, batch=500)
    return data_queues

# 需要导入模块: from ichnaea.models import CellShard [as 别名]
# 或者: from ichnaea.models.CellShard import shards [as 别名]
def write_stations_to_csv(session, path, start_time=None, end_time=None):
    where = 'radio != 1 AND lat IS NOT NULL AND lon IS NOT NULL'
    if None not in (start_time, end_time):
        where = where + ' AND modified >= "%s" AND modified < "%s"'
        fmt = '%Y-%m-%d %H:%M:%S'
        where = where % (start_time.strftime(fmt), end_time.strftime(fmt))

    header_row = [
        'radio', 'mcc', 'net', 'area', 'cell', 'unit',
        'lon', 'lat', 'range', 'samples', 'changeable',
        'created', 'updated', 'averageSignal',
    ]
    header_row = ','.join(header_row) + '\n'

    tables = [shard.__tablename__ for shard in CellShard.shards().values()]
    stmt = '''SELECT
    CONCAT_WS(",",
        CASE radio
            WHEN 0 THEN "GSM"
            WHEN 2 THEN "UMTS"
            WHEN 3 THEN "LTE"
            ELSE ""
        END,
        `mcc`,
        `mnc`,
        `lac`,
        `cid`,
        COALESCE(`psc`, ""),
        ROUND(`lon`, 7),
        ROUND(`lat`, 7),
        COALESCE(`radius`, "0"),
        COALESCE(`samples`, "0"),
        "1",
        COALESCE(UNIX_TIMESTAMP(`created`), ""),
        COALESCE(UNIX_TIMESTAMP(`modified`), ""),
        ""
    ) AS `cell_value`
FROM %s
WHERE %s
ORDER BY `cellid`
LIMIT :l
OFFSET :o
'''

    with util.gzip_open(path, 'w', compresslevel=5) as gzip_wrapper:
        with gzip_wrapper as gzip_file:
            gzip_file.write(header_row)
            for table in tables:
                table_stmt = text(stmt % (table, where))
                offset = 0
                limit = 25000
                while True:
                    rows = session.execute(
                        table_stmt.bindparams(o=offset, l=limit)).fetchall()
                    if rows:
                        buf = '\r\n'.join([row.cell_value for row in rows])
                        if buf:
                            buf += '\r\n'
                        gzip_file.write(buf)
                        offset += limit
                    else:
                        break

# 需要导入模块: from ichnaea.models import CellShard [as 别名]
# 或者: from ichnaea.models.CellShard import shards [as 别名]
def celerybeat_schedule(app_config):
    """Return the celery beat schedule as a dictionary."""

    sections = app_config.sections()

    schedule = {

        # Monitoring

        'monitor-queue-size': {
            'task': 'ichnaea.data.tasks.monitor_queue_size',
            'schedule': timedelta(seconds=60),
            'options': {'expires': 57},
        },
        'monitor-api-users': {
            'task': 'ichnaea.data.tasks.monitor_api_users',
            'schedule': timedelta(seconds=600),
            'options': {'expires': 570},
        },
        'monitor-api-key-limits': {
            'task': 'ichnaea.data.tasks.monitor_api_key_limits',
            'schedule': timedelta(seconds=600),
            'options': {'expires': 570},
        },

        # Statistics

        'update-statcounter': {
            'task': 'ichnaea.data.tasks.update_statcounter',
            'args': (1, ),
            'schedule': crontab(minute=3),
            'options': {'expires': 2700},
        },
        'update-statregion': {
            'task': 'ichnaea.data.tasks.update_statregion',
            'schedule': timedelta(seconds=3600 * 6),
            'options': {'expires': 3600 * 5},
        },

        # Data Pipeline

        'schedule-export-reports': {
            'task': 'ichnaea.data.tasks.schedule_export_reports',
            'schedule': timedelta(seconds=8),
            'options': {'expires': 15},
        },

        'update-cellarea': {
            'task': 'ichnaea.data.tasks.update_cellarea',
            'schedule': timedelta(seconds=8),
            'args': (100, ),
            'options': {'expires': 15},
        },
        'update-cellarea-ocid': {
            'task': 'ichnaea.data.tasks.update_cellarea_ocid',
            'schedule': timedelta(seconds=9),
            'args': (100, ),
            'options': {'expires': 15},
        },

        'update-score': {
            'task': 'ichnaea.data.tasks.update_score',
            'args': (250, ),
            'schedule': timedelta(seconds=9),
            'options': {'expires': 10},
        },

    }

    for shard_id in CellShard.shards().keys():
        schedule.update({
            'update-cell-' + shard_id: {
                'task': 'ichnaea.data.tasks.update_cell',
                'schedule': timedelta(seconds=7),
                'args': (500, shard_id),
                'options': {'expires': 10},
            }
        })

    for shard_id in DataMap.shards().keys():
        schedule.update({
            'update-datamap-' + shard_id: {
                'task': 'ichnaea.data.tasks.update_datamap',
                'args': (500, shard_id),
                'schedule': timedelta(seconds=14),
                'options': {'expires': 20},
            },
        })

    for shard_id in WifiShard.shards().keys():
        schedule.update({
            'update-wifi-' + shard_id: {
                'task': 'ichnaea.data.tasks.update_wifi',
                'schedule': timedelta(seconds=6),
                'args': (500, shard_id),
                'options': {'expires': 10},
            }
        })

    if 'assets' in sections and app_config.get('assets', 'bucket', None):
#.........这里部分代码省略.........

# 需要导入模块: from ichnaea.models import CellShard [as 别名]
# 或者: from ichnaea.models.CellShard import shards [as 别名]
    def tearDown(self):
        for model in CellShard.shards().values():
            self.session.execute(text('drop table %s;' % model.__tablename__))

        self.setup_tables(self.db_rw.engine)
        super(TestDatabaseErrors, self).tearDown()