Python models.StaticDataSourceConfiguration类代码示例

    def handle(self, file_path, *args, **options):
        domain = 'icds-cas'

        data_source_id = StaticDataSourceConfiguration.get_doc_id(domain, PERSON_TABLE_ID)
        config = StaticDataSourceConfiguration.by_id(data_source_id)
        adapter = get_indicator_adapter(config)
        session_helper = connection_manager.get_session_helper(adapter.engine_id)
        person_table_name = get_table_name(domain, PERSON_TABLE_ID)
        awc_location_table_name = get_table_name(domain, AWC_LOCATION_TABLE_ID)
        session = session_helper.Session

        with open(
            os.path.join(os.path.dirname(__file__), 'sql_scripts', 'nos_of_deaths.sql'),
            encoding='utf-8'
        ) as f:
            sql_script = f.read()
            rows = session.execute(
                sql_script % {
                    'person_table_name': person_table_name,
                    'awc_location_table_name': awc_location_table_name
                }
            )

            with open(file_path, 'w', encoding='utf-8') as file_object:
                writer = csv.writer(file_object)
                writer.writerow([
                    'State',
                    'District',
                    'AWC',
                    'Month',
                    'Deaths',
                ])
                writer.writerows(rows)

 def setUpClass(cls):
     super(BaseICDSDatasourceTest, cls).setUpClass()
     cls._call_center_domain_mock.start()
     cls.static_datasource = StaticDataSourceConfiguration.wrap(
         cls.get_json(cls.datasource_filename)
     )
     cls.domain = cls.static_datasource.domains[0]
     cls.datasource = StaticDataSourceConfiguration._get_datasource_config(
         cls.static_datasource,
         cls.domain,
     )
     cls.casefactory = CaseFactory(domain=cls.domain)
     cls.adapter = get_indicator_adapter(cls.datasource)
     cls.adapter.rebuild_table()

def rebuild_indicators(indicator_config_id):
    is_static = indicator_config_id.startswith(StaticDataSourceConfiguration._datasource_id_prefix)
    if is_static:
        config = StaticDataSourceConfiguration.by_id(indicator_config_id)
    else:
        config = DataSourceConfiguration.get(indicator_config_id)
        # Save the start time now in case anything goes wrong. This way we'll be
        # able to see if the rebuild started a long time ago without finishing.
        config.meta.build.initiated = datetime.datetime.utcnow()
        config.save()

    adapter = IndicatorSqlAdapter(config)
    adapter.rebuild_table()

    couchdb = _get_db(config.referenced_doc_type)
    relevant_ids = get_doc_ids(config.domain, config.referenced_doc_type, database=couchdb)

    for doc in iter_docs(couchdb, relevant_ids, chunksize=500):
        try:
            # save is a noop if the filter doesn't match
            adapter.save(doc)
        except DataError as e:
            logging.exception("problem saving document {} to table. {}".format(doc["_id"], e))

    if not is_static:
        config.meta.build.finished = True
        config.save()

    def handle(self, domains, **options):
        if not domains and not options['all']:
            raise CommandError('Specify specific domains or --all')

        all_domains = [domain.name for domain in get_call_center_domains() if domain.use_fixtures]
        if domains:
            for domain in domains:
                assert domain in all_domains, "Domain '{}' is not a Call Center domain".format(domain)
        else:
            domains = all_domains

        threshold = options['threshold']
        domain_stats = get_call_center_data_source_stats(domains)
        for domain in domains:
            stats = domain_stats[domain]
            print('Checking domain:', domain)
            if stats.error:
                print('Error getting stats:\n', stats.error)
                continue

            for stat in stats.iter_data_source_stats():
                diff = math.fabs(stat.ucr_percent - stat.es_percent)
                if diff > threshold:
                    print("rebuilding data source '{}' in domain '{}': diff = {}".format(
                        stat.name, domain, diff
                    ))
                    try:
                        rebuild_indicators(
                            StaticDataSourceConfiguration.get_doc_id(domain, TABLE_IDS[stat.name])
                        )
                    except Exception as e:
                        sys.stderr.write("Error rebuilding data source '{}' in domain '{}':\n{}".format(
                            stat.name, domain, e
                        ))

def setUpModule():
    if isinstance(Domain.get_db(), Mock):
        # needed to skip setUp for javascript tests thread on Travis
        return

    _call_center_domain_mock = mock.patch(
        'corehq.apps.callcenter.data_source.call_center_data_source_configuration_provider'
    )
    _call_center_domain_mock.start()

    domain = create_domain('champ-cameroon')
    with override_settings(SERVER_ENVIRONMENT='production'):

        configs = StaticDataSourceConfiguration.by_domain(domain.name)
        adapters = [get_indicator_adapter(config) for config in configs]

        for adapter in adapters:
            adapter.build_table()

        engine = connection_manager.get_engine(UCR_ENGINE_ID)
        metadata = sqlalchemy.MetaData(bind=engine)
        metadata.reflect(bind=engine, extend_existing=True)
        path = os.path.join(os.path.dirname(__file__), 'fixtures')
        for file_name in os.listdir(path):
            with open(os.path.join(path, file_name), encoding='utf-8') as f:
                table_name = get_table_name(domain.name, file_name[:-4])
                table = metadata.tables[table_name]
                postgres_copy.copy_from(
                    f, table, engine, format='csv' if six.PY3 else b'csv',
                    null='' if six.PY3 else b'', header=True
                )
    _call_center_domain_mock.stop()

    def handle(self, **options):
        data_sources = list(DataSourceConfiguration.all())
        data_sources.extend(list(StaticDataSourceConfiguration.all()))

        engine_ids = self._get_engine_ids(data_sources, options.get('engine_id'))

        tables_to_remove_by_engine = defaultdict(list)
        for engine_id in engine_ids:
            engine = connection_manager.get_engine(engine_id)
            with engine.begin() as connection:
                migration_context = get_migration_context(connection, include_object=_include_object)
                raw_diffs = compare_metadata(migration_context, metadata)

            diffs = reformat_alembic_diffs(raw_diffs)
            tables_to_remove_by_engine[engine_id] = [
                diff.table_name for diff in diffs
                if diff.type == 'remove_table'
            ]

        for engine_id, tablenames in tables_to_remove_by_engine.items():
            engine = connection_manager.get_engine(engine_id)
            for tablename in tablenames:
                with engine.begin() as connection:
                    try:
                        result = connection.execute(
                            'SELECT COUNT(*), MAX(inserted_at) FROM "{tablename}"'.format(tablename=tablename)
                        )
                    except Exception:
                        print(tablename, "no inserted_at column, probably not UCR")
                    else:
                        print(tablename, result.fetchone())

def get_datasources_for_domain(domain, referenced_doc_type=None, include_static=False, include_aggregate=False):
    from corehq.apps.userreports.models import DataSourceConfiguration, StaticDataSourceConfiguration
    key = [domain]
    if referenced_doc_type:
        key.append(referenced_doc_type)
    datasources = sorted(
        DataSourceConfiguration.view(
            'userreports/data_sources_by_build_info',
            startkey=key,
            endkey=key + [{}],
            reduce=False,
            include_docs=True
        ),
        key=lambda config: config.display_name or '')

    if include_static:
        static_ds = StaticDataSourceConfiguration.by_domain(domain)
        if referenced_doc_type:
            static_ds = [ds for ds in static_ds if ds.referenced_doc_type == referenced_doc_type]
        datasources.extend(sorted(static_ds, key=lambda config: config.display_name))

    if include_aggregate:
        from corehq.apps.aggregate_ucrs.models import AggregateTableDefinition
        datasources.extend(AggregateTableDefinition.objects.filter(domain=domain).all())
    return datasources

def _make_data_source_for_domain(data_source_json, domain_name):
    from corehq.apps.userreports.models import StaticDataSourceConfiguration
    from corehq.apps.userreports.models import DataSourceConfiguration

    doc = deepcopy(data_source_json)
    doc['domain'] = domain_name
    doc['_id'] = StaticDataSourceConfiguration.get_doc_id(domain_name, doc['table_id'])
    return DataSourceConfiguration.wrap(doc)

    def handle(self, domain, **options):
        tables = StaticDataSourceConfiguration.by_domain(domain)
        tables.extend(DataSourceConfiguration.by_domain(domain))

        print("Rebuilding {} tables".format(len(tables)))

        for table in tables:
            tasks.rebuild_indicators(table._id)

def _shared_context(domain):
    static_reports = list(StaticReportConfiguration.by_domain(domain))
    static_data_sources = list(StaticDataSourceConfiguration.by_domain(domain))
    return {
        'domain': domain,
        'reports': ReportConfiguration.by_domain(domain) + static_reports,
        'data_sources': DataSourceConfiguration.by_domain(domain) + static_data_sources,
    }

 def test_get_all(self):
     with override_settings(STATIC_DATA_SOURCES=[self.get_path('sample_static_data_source', 'json')]):
         all = list(StaticDataSourceConfiguration.all())
         self.assertEqual(2, len(all))
         example, dimagi = all
         self.assertEqual('example', example.domain)
         self.assertEqual('dimagi', dimagi.domain)
         for config in all:
             self.assertEqual('all_candidates', config.table_id)

 def __init__(self, domain, *args, **kwargs):
     self.domain = domain
     standard_sources = DataSourceConfiguration.by_domain(self.domain)
     custom_sources = list(StaticDataSourceConfiguration.by_domain(domain))
     available_data_sources = standard_sources + custom_sources
     super(ReportDataSourceField, self).__init__(
         choices=[(src._id, src.display_name) for src in available_data_sources],
         *args, **kwargs
     )

    def copy_ucr_datasources(self):
        datasource_map = {}
        datasources = get_datasources_for_domain(self.existing_domain)
        for datasource in datasources:
            datasource.meta.build.finished = False
            datasource.meta.build.initiated = None

            old_id, new_id = self.save_couch_copy(datasource, self.new_domain)
            datasource_map[old_id] = new_id
        for static_datasource in StaticDataSourceConfiguration.by_domain(self.existing_domain):
            table_id = static_datasource.get_id.replace(
                StaticDataSourceConfiguration._datasource_id_prefix + self.existing_domain + "-", ""
            )
            new_id = StaticDataSourceConfiguration.get_doc_id(self.new_domain, table_id)
            # check that new datasource is in new domain's list of static datasources
            StaticDataSourceConfiguration.by_id(new_id)
            datasource_map[static_datasource.get_id] = new_id
        return datasource_map

 def main_context(self):
     static_reports = list(StaticReportConfiguration.by_domain(self.domain))
     static_data_sources = list(StaticDataSourceConfiguration.by_domain(self.domain))
     context = super(BaseUserConfigReportsView, self).main_context
     context.update({
         'reports': ReportConfiguration.by_domain(self.domain) + static_reports,
         'data_sources': DataSourceConfiguration.by_domain(self.domain) + static_data_sources,
     })
     return context

def _data_sources_by_engine_id():
    by_engine_id = defaultdict(list)
    for ds in StaticDataSourceConfiguration.all():
        ds_engine_id = ds['engine_id']
        by_engine_id[ds_engine_id].append(ds)

    for ds in DataSourceConfiguration.all():
        ds_engine_id = ds['engine_id']
        by_engine_id[ds_engine_id].append(ds)

    return by_engine_id