Python loader.OnsLoader类代码示例

    def setup_class(self):
        super(TestOnsLoadMissingDept, self).setup_class()

        try:
            self.orig_pkg_dict = {
                 "name": u"measuring_subjective_wellbeing_in_the_uk",
                 "title": "Measuring Subjective Wellbeing in the UK",
                 "notes": "This report reviews:\n\nWhat is subjective wellbeing and why should we measure it?\n\nHow subjective wellbeing is currently measured in the UK - what subjective wellbeing questions are already being asked on major social surveys in the UK\n\nThe potential uses of subjective wellbeing data collected via these surveys\n\n\nIt concludes that subjective wellbeing is a valid construct that can be measured reliably. This is the first output of ONS' work on subjective wellbeing.\n\nSource agency: Office for National Statistics\n\nDesignation: Supporting material\n\nLanguage: English\n\nAlternative title: Working Paper: Measuring Subjective Wellbeing in the UK",
                 "license_id": "ukcrown-withrights",
                 "tags": ["communities", "health-well-being-and-care", "people-and-places", "societal-wellbeing", "subjective-wellbeing-subjective-well-being-objective-measures-subjective-measures", "well-being"],
                 "groups": ['office-for-national-statistics'],
                 "extras": {"geographic_coverage": "111100: United Kingdom (England, Scotland, Wales, Northern Ireland)", "geographic_granularity": "UK and GB", "external_reference": "ONSHUB", "temporal_granularity": "", "date_updated": "", "precision": "", "temporal_coverage_to": "", "temporal_coverage_from": "", "national_statistic": "no", "import_source": "ONS-ons_data_7_days_to_2010-09-17", "update_frequency": "", "date_released": "2010-09-14", "categories": "People and Places"},
                "resources": [{"url": "http://www.ons.gov.uk/about-statistics/measuring-equality/wellbeing/news-and-events/index.html", "format": "", "description": "2010", "extras":{"hub-id":"77-31166"}}],
                 }
            CreateTestData.create_arbitrary([self.orig_pkg_dict])

            # same data is imported, but should find record and add department
            importer_ = importer.OnsImporter(sample_filepath(5), self.testclient)
            self.pkg_dict = [pkg_dict for pkg_dict in importer_.pkg_dict()][0]
            loader = OnsLoader(self.testclient)
            self.res = loader.load_package(self.pkg_dict)
        except:
            # ensure that mock_drupal is destroyed
            MockDrupalCase.teardown_class()
            model.repo.rebuild_db()
            raise

    def command(self):
        ApiCommand.command(self)
        XmlRpcCommand.command(self)

        if self.options.days:
            self.options.days = int(self.options.days)
        if self.options.start_date:
            self.options.start_date = self.parse_date(self.options.start_date)
        if self.options.end_date:
            self.options.end_date = self.parse_date(self.options.end_date)

        if self.options.days or self.options.start_date or self.options.end_date:
            data_filepaths = OnsData.download_flexible(
                days=self.options.days,
                start_date=self.options.start_date,
                end_date=self.options.end_date,
                ons_cache_dir=self.options.ons_cache_dir,
            )

        elif self.options.all_time:
            data_filepaths = OnsData.download_all()
        else:
            self.parser.error("Please specify a time period")

        importer = OnsImporter(filepaths=data_filepaths, xmlrpc_settings=self.xmlrpc_settings)
        loader = OnsLoader(self.client)

        loader.load_packages(importer.pkg_dict())

    def command(self):
        from ckanext.dgu.ons.downloader import OnsData, ONS_DEFAULT_CACHE_PATH
        from ckanext.dgu.ons.importer import OnsImporter
        from ckanext.dgu.ons.loader import OnsLoader

        ApiCommand.command(self)
        log = logging.getLogger(__name__)

        try:
            if self.options.days:
                self.options.days = int(self.options.days)
            if self.options.start_date:
                self.options.start_date = self.parse_date(self.options.start_date)
            if self.options.end_date:
                self.options.end_date = self.parse_date(self.options.end_date)
            if self.options.month:
                self.options.month = self.parse_month(self.options.month)
            if self.options.months_since:
                self.options.months_since = self.parse_month(self.options.months_since)
            if not self.options.ons_cache_dir:
                self.options.ons_cache_dir = ONS_DEFAULT_CACHE_PATH

            if self.options.days or \
                self.options.start_date or \
                self.options.end_date:
                data_filepaths = OnsData.download_flexible(
                    days=self.options.days,
                    start_date=self.options.start_date,
                    end_date=self.options.end_date,
                    ons_cache_dir=self.options.ons_cache_dir)

            elif self.options.month:
                data_filepaths = OnsData.download_month(year=self.options.month.year,
                                                        month=self.options.month.month)
            elif self.options.months_since:
                data_filepaths = OnsData.download_months_since(
                    year=self.options.months_since.year,
                    month=self.options.months_since.month,
                    force_download=self.options.force_download)
            elif self.options.all_time:
                data_filepaths = OnsData.download_all(force_download=self.options.force_download)
            else:
                self.parser.error('Please specify a time period')

            filter_ = {}
            if self.options.publisher:
                filter_['publisher'] = self.options.publisher

            stats = StatsList()
            importer = OnsImporter(filepaths=data_filepaths,
                                   ckanclient=self.client, stats=stats,
                                   filter_=filter_)
            loader = OnsLoader(self.client, stats)

            loader.load_packages(importer.pkg_dict())
            log.info('Summary:\n' + stats.report())
        except:
            # Any problem, make sure it gets logged
            log.exception('ONS Loader exception')
            raise

 def setup_class(self):
     super(TestReloadUnknownPublisher, self).setup_class()
     for filepath in (sample_filepath('10'), sample_filepath('10')):
         importer_ = importer.OnsImporter(filepath)
         pkg_dicts = [pkg_dict for pkg_dict in importer_.pkg_dict()]
         assert_equal(len(pkg_dicts), 1)
         loader = OnsLoader(self.testclient)
         res = loader.load_packages(pkg_dicts)
         assert res['num_errors'] == 0, res

 def setup_class(self):
     super(TestOnsLoadSeries, self).setup_class()
     for filepath in [sample_filepath('4a'), sample_filepath('4b')]:
         importer_ = importer.OnsImporter(filepath)
         pkg_dicts = [pkg_dict for pkg_dict in importer_.pkg_dict()]
         for pkg_dict in pkg_dicts:
             assert pkg_dict['title'] == 'Regional Labour Market Statistics', pkg_dict
             assert_equal(strip_organisation_id(pkg_dict['extras']['published_by']),
                          'Office for National Statistics [some_number]')
             assert '2010-08-' in pkg_dict['extras']['date_released'], pkg_dict
             assert pkg_dict['extras']['date_updated'] == '', pkg_dict
         loader = OnsLoader(self.testclient)
         res = loader.load_packages(pkg_dicts)
         assert res['num_errors'] == 0, res

    def test_load(self):
        user = model.User.by_name(u'testsysadmin')
        assert user
        testclient_admin = WsgiCkanClient(self.app, api_key=user.apikey)
        loader = OnsLoader(testclient_admin)
        print self.pkg_dict
        self.res = loader.load_package(self.pkg_dict)
        self.name = self.orig_pkg_dict['name']
        self.decoy_name = self.deleted_decoy_pkg_dict['name']
        self.num_resources_originally = len(self.orig_pkg_dict['resources'])

        names = [pkg.name for pkg in model.Session.query(model.Package).all()]
        assert_equal(set(names), set((self.name, self.decoy_name)))
        pkg = model.Package.by_name(self.name)
        assert pkg
        assert_equal(len(pkg.resources), self.num_resources_originally + 1)

    def command(self):
        from ckanext.dgu.ons.downloader import OnsData, ONS_DEFAULT_CACHE_PATH
        from ckanext.dgu.ons.importer import OnsImporter
        from ckanext.dgu.ons.loader import OnsLoader

        ApiCommand.command(self)

        if self.options.days:
            self.options.days = int(self.options.days)
        if self.options.start_date:
            self.options.start_date = self.parse_date(self.options.start_date)
        if self.options.end_date:
            self.options.end_date = self.parse_date(self.options.end_date)
        if self.options.month:
            self.options.month = self.parse_month(self.options.month)
        if self.options.months_since:
            self.options.months_since = self.parse_month(self.options.months_since)
        if not self.options.ons_cache_dir:
            self.options.ons_cache_dir = ONS_DEFAULT_CACHE_PATH

        if self.options.days or \
               self.options.start_date or \
               self.options.end_date:
            data_filepaths = OnsData.download_flexible(
                days=self.options.days,
                start_date=self.options.start_date,
                end_date=self.options.end_date,
                ons_cache_dir=self.options.ons_cache_dir)

        elif self.options.month:
            data_filepaths = OnsData.download_month(year=self.options.month.year,
                                                    month=self.options.month.month)
        elif self.options.months_since:
            data_filepaths = OnsData.download_months_since(
                year=self.options.months_since.year,
                month=self.options.months_since.month,
                force_download=self.options.force_download)
        elif self.options.all_time:
            data_filepaths = OnsData.download_all(force_download=self.options.force_download)
        else:
            self.parser.error('Please specify a time period')

        importer = OnsImporter(filepaths=data_filepaths, ckanclient=self.client)
        loader = OnsLoader(self.client)

        loader.load_packages(importer.pkg_dict())

    def command(self):
        from ckanext.dgu.ons.downloader import OnsData, ONS_DEFAULT_CACHE_PATH
        from ckanext.dgu.ons.importer import OnsImporter
        from ckanext.dgu.ons.loader import OnsLoader

        ApiCommand.command(self)
        XmlRpcCommand.command(self)

        if self.options.days:
            self.options.days = int(self.options.days)
        if self.options.start_date:
            self.options.start_date = self.parse_date(self.options.start_date)
        if self.options.end_date:
            self.options.end_date = self.parse_date(self.options.end_date)
        if not self.options.ons_cache_dir:
            self.options.ons_cache_dir = ONS_DEFAULT_CACHE_PATH

        if self.options.days or \
               self.options.start_date or \
               self.options.end_date:
            data_filepaths = OnsData.download_flexible(
                days=self.options.days,
                start_date=self.options.start_date,
                end_date=self.options.end_date,
                ons_cache_dir=self.options.ons_cache_dir)

        elif self.options.all_time:
            data_filepaths = OnsData.download_all()
        else:
            self.parser.error('Please specify a time period')

        importer = OnsImporter(filepaths=data_filepaths,
                               xmlrpc_settings=self.xmlrpc_settings)
        loader = OnsLoader(self.client)

        loader.load_packages(importer.pkg_dict())

class TestOnsLoadBasic(OnsLoaderBase):
    lots_of_publishers = True
    
    @classmethod
    def setup_class(self):
        super(TestOnsLoadBasic, self).setup_class()
        try:
            user = model.User.by_name(u'annafan')
            assert user
            test_ckan_client = WsgiCkanClient(self.app, api_key=user.apikey)
            importer_ = importer.OnsImporter(sample_filepath(''), test_ckan_client)
            self.pkg_dicts = [pkg_dict for pkg_dict in importer_.pkg_dict()]

            self.loader = OnsLoader(test_ckan_client)
            self.res = self.loader.load_packages(self.pkg_dicts)
            assert self.res['num_errors'] == 0, self.res
        except Exception:
            # ensure that mock_drupal is destroyed
            MockDrupalCase.teardown_class()
            model.repo.rebuild_db()
            raise

    def test_0_search_options(self):
        field_keys = ['title', 'groups']

        pkg_dict = {'title':'titleA',
                    'groups':['Department for Children, Schools and Families']}
        opts = self.loader._get_search_options(field_keys, pkg_dict)
        assert_equal(opts, [{'groups': 'Department for Children, Schools and Families', 'title': 'titleA'}])

    def test_1_hub_id_extraction(self):
        def assert_id(hub_id_value, expected_id):
            resource = {'description':'Some description',
                        'hub-id':hub_id_value}
            result = self.loader._get_hub_id(resource)
            assert_equal(result, expected_id)
        assert_id("119-46440",
                  "119-46440")

    def test_2_date_choose(self):
        def assert_id(date1, date2, earlier_or_later, expected_date_index):
            dates = (date1, date2)
            pkg0 = {'extras':{'date': date1}}
            result = self.loader._choose_date(pkg0, date2,
                                              earlier_or_later,
                                              'date')
            if not expected_date_index:
                assert_equal(result, expected_date_index)
            else:
                assert_equal(result, dates[expected_date_index - 1])
        assert_id('2010-12-01', '2010-12-02', 'earlier', 1)
        assert_id('2010-12-01', '2010-12-02', 'later', 2)
        assert_id('2010-12-02', '2010-12-01', 'earlier', 2)
        assert_id('2010-12-02', '2010-12-01', 'later', 1)
        assert_id('', '2010-12-02', 'earlier', 2)
        assert_id('2010-12-01', '', 'later', 1)
        assert_id('', '', 'earlier', None)

    def test_fields(self):
        q = model.Session.query(model.Package)
        names = [pkg.name for pkg in q.all()]
        pkg1 = model.Package.by_name(u'uk_official_holdings_of_international_reserves')
        cereals = model.Package.by_name(u'cereals_and_oilseeds_production_harvest')
        custody = model.Package.by_name(u'end_of_custody_licence_release_and_recalls')
        probation = model.Package.by_name(u'probation_statistics_brief')
        assert pkg1, names
        assert cereals, names
        assert custody, names
        assert probation, names
        assert pkg1.title == 'UK Official Holdings of International Reserves', pkg1.title
        assert pkg1.notes.startswith("Monthly breakdown for government's net reserves, detailing gross reserves and gross liabilities."), pkg1.notes
        assert len(pkg1.resources) == 1, pkg1.resources
        assert pkg1.resources[0].url == 'http://www.hm-treasury.gov.uk/national_statistics.htm', pkg1.resources[0]
        assert_equal(pkg1.resources[0].description, 'December 2009')
        assert_equal(pkg1.resources[0].extras['hub-id'], '119-36345')
        assert len(custody.resources) == 2, custody.resources
        assert custody.resources[0].url == 'http://www.justice.gov.uk/publications/endofcustodylicence.htm', custody.resources[0]
        assert_equal(custody.resources[0].description, 'November 2009')
        assert_equal(custody.resources[0].extras['hub-id'], '119-36836')
        assert custody.resources[1].url == 'http://www.justice.gov.uk/publications/endofcustodylicence.htm', custody.resources[0]
        assert_equal(custody.resources[1].description, 'December 2009')
        assert_equal(custody.resources[1].extras['hub-id'], '119-36838')
        assert pkg1.extras['date_released'] == u'2010-01-06', pkg1.extras['date_released']
        assert probation.extras['date_released'] == u'2010-01-04', probation.extras['date_released']
        assert_equal(group_names(pkg1), [u"her-majestys-treasury"])
        assert_equal(group_names(cereals), [u"department-for-environment-food-and-rural-affairs"])
        assert_equal(group_names(custody), [u"department-of-justice"])
        assert u"Source agency: HM Treasury" in pkg1.notes, pkg1.notes
        assert pkg1.extras['categories'] == 'Economy', pkg1.extras['category']
        assert_equal(pkg1.extras['geographic_coverage'], '111100: United Kingdom (England, Scotland, Wales, Northern Ireland)')
        assert pkg1.extras['national_statistic'] == 'no', pkg1.extras['national_statistic']
        assert cereals.extras['national_statistic'] == 'yes', cereals.extras['national_statistic']
        assert custody.extras['national_statistic'] == 'no', custody.extras['national_statistic']
        assert 'Designation: Official Statistics not designated as National Statistics' in custody.notes
        assert_equal(pkg1.extras['geographic_granularity'], 'UK and GB')
        assert 'Language: English' in pkg1.notes, pkg1.notes
        def check_tags(pkg, tags_list):            
            pkg_tags = [tag.name for tag in pkg.get_tags()]
            for tag in tags_list:
                assert tag in pkg_tags, "Couldn't find tag '%s' in tags: %s" % (tag, pkg_tags)
#.........这里部分代码省略.........