Python Datasets.update_database方法代码示例

# 需要导入模块: from dlstats.fetchers._commons import Datasets [as 别名]
# 或者: from dlstats.fetchers._commons.Datasets import update_database [as 别名]
    def upsert_dataset(self, dataset_code, datas=None):
        
        start = time.time()
        
        logger.info("upsert dataset[%s] - START" % (dataset_code))
        
        if not DATASETS.get(dataset_code):
            raise Exception("This dataset is unknown" + dataset_code)
        
        dataset = Datasets(provider_name=self.provider_name, 
                           dataset_code=dataset_code, 
                           name=DATASETS[dataset_code]['name'], 
                           doc_href=DATASETS[dataset_code]['doc_href'],
                           fetcher=self)
        
        fetcher_data = OECD_Data(dataset)
        dataset.series.data_iterator = fetcher_data
        dataset.update_database()

        end = time.time() - start
        logger.info("upsert dataset[%s] - END-BEFORE-METAS - time[%.3f seconds]" % (dataset_code, end))

        self.update_metas(dataset_code)
        
        end = time.time() - start
        logger.info("upsert dataset[%s] - END - time[%.3f seconds]" % (dataset_code, end))

# 需要导入模块: from dlstats.fetchers._commons import Datasets [as 别名]
# 或者: from dlstats.fetchers._commons.Datasets import update_database [as 别名]
    def test_process_series_data(self):        

        # nosetests -s -v dlstats.tests.fetchers.test__commons:DBSeriesTestCase.test_process_series_data

        self._collections_is_empty()
    
        provider_name = "p1"
        dataset_code = "d1"
        dataset_name = "d1 name"
    
        f = Fetcher(provider_name=provider_name, 
                    db=self.db)

        d = Datasets(provider_name=provider_name, 
                    dataset_code=dataset_code,
                    name=dataset_name,
                    last_update=datetime.now(),
                    doc_href="http://www.example.com",
                    fetcher=f, 
                    is_load_previous_version=False)
        d.dimension_list.update_entry("Scale", "Billions", "Billions")
        d.dimension_list.update_entry("Country", "AFG", "AFG")
        
        s = Series(provider_name=f.provider_name, 
                   dataset_code=dataset_code, 
                   last_update=datetime(2013,10,28), 
                   bulk_size=1, 
                   fetcher=f)
        
        datas = FakeDatas(provider_name=provider_name, 
                          dataset_code=dataset_code,
                          fetcher=f)
        s.data_iterator = datas
        
        d.series = s
        d.update_database()        
        
        '''Count All series'''
        self.assertEqual(self.db[constants.COL_SERIES].count(), datas.max_record)

        '''Count series for this provider and dataset'''
        series = self.db[constants.COL_SERIES].find({'provider_name': f.provider_name, 
                                                     "dataset_code": dataset_code})
        self.assertEqual(series.count(), datas.max_record)

        tags.update_tags(self.db, 
                    provider_name=f.provider_name, dataset_code=dataset_code,  
                    col_name=constants.COL_SERIES)        

        '''Count series for this provider and dataset and in keys[]'''
        series = self.db[constants.COL_SERIES].find({'provider_name': f.provider_name, 
                                                     "dataset_code": dataset_code,
                                                     "key": {"$in": datas.keys}})
        
        self.assertEqual(series.count(), datas.max_record)
        

        for doc in series:
            self.assertTrue("tags" in doc)
            self.assertTrue(len(doc['tags']) > 0)

# 需要导入模块: from dlstats.fetchers._commons import Datasets [as 别名]
# 或者: from dlstats.fetchers._commons.Datasets import update_database [as 别名]
 def upsert_sna(self, url, dataset_code):
     dataset = Datasets(self.provider_name,dataset_code,
                        fetcher=self)
     sna_data = EsriData(dataset,url)
     dataset.name = self.dataset_name[dataset_code]
     dataset.doc_href = 'http://www.esri.cao.go.jp/index-e.html'
     dataset.last_update = sna_data.release_date
     dataset.series.data_iterator = sna_data
     dataset.update_database()

# 需要导入模块: from dlstats.fetchers._commons import Datasets [as 别名]
# 或者: from dlstats.fetchers._commons.Datasets import update_database [as 别名]
 def upsert_gem(self, dataset_code):
     d = DATASETS[dataset_code]
     url = d['url']
     dataset = Datasets(provider_name=self.provider_name, 
                        dataset_code=dataset_code, 
                        name=d['name'], 
                        doc_href=d['doc_href'], 
                        fetcher=self)
     gem_data = GemData(dataset, url)
     dataset.last_update = gem_data.release_date
     dataset.series.data_iterator = gem_data
     dataset.update_database()

# 需要导入模块: from dlstats.fetchers._commons import Datasets [as 别名]
# 或者: from dlstats.fetchers._commons.Datasets import update_database [as 别名]
 def load_datasets_update(self):
     for d in self._parse_agenda():
         if d['dataflow_key'] in self.datasets_filter:
             dataset = Datasets(provider_name=self.provider_name,
                                dataset_code=d['dataflow_key'],
                                name=d['name'],
                                last_update=d['last_update'],
                                fetcher=self)
             url = d['url']                
             dataset.series.data_iterator = BDF_Data(dataset, url)
             dataset.update_database()    
             msg = "get update - provider[%s] - dataset[%s] - last-update-dataset[%s]"    
             logger.info(msg % (self.provider_name, d['dataflow_key'], d['last_update']))                  

# 需要导入模块: from dlstats.fetchers._commons import Datasets [as 别名]
# 或者: from dlstats.fetchers._commons.Datasets import update_database [as 别名]
    def _common_tests(self):
        
        self._collections_is_empty()
        
        url = DATASETS[self.dataset_code]['url']

        self.filepath = get_filepath(self.dataset_code)
        self.assertTrue(os.path.exists(self.filepath))
        
        httpretty.register_uri(httpretty.GET, 
                               url,
                               body=mock_streaming(self.filepath),
                               status=200,
                               content_type='application/octet-stream;charset=UTF-8',
                               streaming=True)
        
        # provider.update_database
        self.fetcher.provider.update_database()
        provider = self.db[constants.COL_PROVIDERS].find_one({"name": self.fetcher.provider_name})
        self.assertIsNotNone(provider)
        
        # upsert_data_tree
        self.fetcher.upsert_data_tree()
        provider = self.db[constants.COL_PROVIDERS].find_one({"name": self.fetcher.provider_name}) 
        self.assertIsNotNone(provider['data_tree'])
        
        dataset = Datasets(provider_name=self.fetcher.provider_name, 
                           dataset_code=self.dataset_code, 
                           name=DATASETS[self.dataset_code]['name'], 
                           doc_href=DATASETS[self.dataset_code]['doc_href'], 
                           fetcher=self.fetcher)

        fetcher_data = bis.BIS_Data(dataset,
                                    url=url, 
                                    filename=DATASETS[self.dataset_code]['filename'],
                                    store_filepath=os.path.dirname(self.filepath))
        
        dataset.series.data_iterator = fetcher_data
        dataset.update_database()

        self.dataset = self.db[constants.COL_DATASETS].find_one({'provider_name': self.fetcher.provider_name, 
                                                            "dataset_code": self.dataset_code})
        
        self.assertIsNotNone(self.dataset)
        
        self.assertEqual(len(self.dataset["dimension_list"]), DATASETS[self.dataset_code]["dimensions_count"])
        
        series = self.db[constants.COL_SERIES].find({'provider_name': self.fetcher.provider_name, 
                                                     "dataset_code": self.dataset_code})
        self.assertEqual(series.count(), SERIES_COUNT)

# 需要导入模块: from dlstats.fetchers._commons import Datasets [as 别名]
# 或者: from dlstats.fetchers._commons.Datasets import update_database [as 别名]
 def upsert_dataset(self, dataset_code, sheet):    
     start = time.time()
     logger.info("upsert dataset[%s] - START" % (dataset_code))
     
     dataset = Datasets(self.provider_name,dataset_code,
                        fetcher=self)
     bea_data = BeaData(dataset,self.url, sheet)
     dataset.name = dataset_code
     dataset.doc_href = 'http://www.bea.gov/newsreleases/national/gdp/gdpnewsrelease.htm'
     dataset.last_update = bea_data.release_date
     dataset.series.data_iterator = bea_data
     dataset.update_database()
     self.update_metas(dataset_code)
     end = time.time() - start
     logger.info("upsert dataset[%s] - END - time[%.3f seconds]" % (dataset_code, end))

# 需要导入模块: from dlstats.fetchers._commons import Datasets [as 别名]
# 或者: from dlstats.fetchers._commons.Datasets import update_database [as 别名]
    def test_not_recordable_dataset(self):

        # nosetests -s -v dlstats.tests.fetchers.test__commons:DBDatasetTestCase.test_not_recordable_dataset

        self._collections_is_empty()

        f = Fetcher(provider_name="p1", 
                    db=self.db)

        d = Datasets(provider_name="p1", 
                    dataset_code="d1",
                    name="d1 Name",
                    last_update=datetime.now(),
                    doc_href="http://www.example.com",
                    fetcher=f, 
                    is_load_previous_version=False)
        d.dimension_list.update_entry("Scale", "Billions", "Billions")
        d.dimension_list.update_entry("country", "AFG", "AFG")
        
        class EmptySeriesIterator():
            def __next__(self):
                raise StopIteration            

        datas = EmptySeriesIterator()
        d.series.data_iterator = datas

        id = d.update_database()
        self.assertIsNone(id)
        
        self.assertEqual(self.db[constants.COL_DATASETS].count(), 0)

# 需要导入模块: from dlstats.fetchers._commons import Datasets [as 别名]
# 或者: from dlstats.fetchers._commons.Datasets import update_database [as 别名]
    def test_unique_constraint(self):

        # nosetests -s -v dlstats.tests.fetchers.test__commons:DBDatasetTestCase.test_unique_constraint
    
        self._collections_is_empty()
        
        f = Fetcher(provider_name="p1", 
                    db=self.db)

        d = Datasets(provider_name="p1", 
                    dataset_code="d1",
                    name="d1 Name",
                    last_update=datetime.now(),
                    doc_href="http://www.example.com",
                    fetcher=f, 
                    is_load_previous_version=False)
        d.dimension_list.update_entry("Country", "AFG", "AFG")
        d.dimension_list.update_entry("Scale", "Billions", "Billions")

        datas = FakeDatas(provider_name="p1", 
                          dataset_code="d1",
                          fetcher=f)
        d.series.data_iterator = datas

        result = d.update_database()
        self.assertIsNotNone(result)
        
        self.assertEqual(self.db[constants.COL_DATASETS].count(), 1)
                        
        with self.assertRaises(DuplicateKeyError):
            existing_dataset = dict(provider_name="p1", dataset_code="d1")
            self.db[constants.COL_DATASETS].insert(existing_dataset)

# 需要导入模块: from dlstats.fetchers._commons import Datasets [as 别名]
# 或者: from dlstats.fetchers._commons.Datasets import update_database [as 别名]
    def upsert_dataset(self, dataset_code):
        
        start = time.time()
        logger.info("upsert dataset[%s] - START" % (dataset_code))
        
        #TODO: control si existe ou update !!!

        dataset = Datasets(provider_name=self.provider_name, 
                           dataset_code=dataset_code,
                           name=DATASETS[dataset_code]['name'],
                           doc_href=DATASETS[dataset_code]['doc_href'],
                           last_update=datetime.now(),
                           fetcher=self)
        
        _data = FED_Data(dataset=dataset, 
                         url=DATASETS[dataset_code]['url'])
        dataset.series.data_iterator = _data
        result = dataset.update_database()
        
        _data = None

        end = time.time() - start
        logger.info("upsert dataset[%s] - END - time[%.3f seconds]" % (dataset_code, end))
        
        return result

# 需要导入模块: from dlstats.fetchers._commons import Datasets [as 别名]
# 或者: from dlstats.fetchers._commons.Datasets import update_database [as 别名]
    def upsert_dataset(self, dataset_code):
        
        settings = self._get_datasets_settings()[dataset_code]
        
        dataset = Datasets(provider_name=self.provider_name, 
                           dataset_code=dataset_code, 
                           name=settings["name"], 
                           doc_href='http://www.bea.gov', 
                           fetcher=self)
        
        url = settings["metadata"]["url"]
        filename = settings["metadata"]["filename"]
        sheet_name = settings["metadata"]["sheet_name"]

        sheet = self._get_sheet(url, filename, sheet_name)
        fetcher_data = BeaData(dataset, url=url, sheet=sheet)
        
        if dataset.last_update and fetcher_data.release_date >= dataset.last_update and not self.force_update: 
            comments = "update-date[%s]" % fetcher_data.release_date
            raise errors.RejectUpdatedDataset(provider_name=self.provider_name,
                                              dataset_code=dataset_code,
                                              comments=comments)
        
        
        dataset.last_update = fetcher_data.release_date
        dataset.series.data_iterator = fetcher_data
        
        return dataset.update_database()

# 需要导入模块: from dlstats.fetchers._commons import Datasets [as 别名]
# 或者: from dlstats.fetchers._commons.Datasets import update_database [as 别名]
    def upsert_dataset(self, dataset_code):
        """Updates data in Database for selected datasets
        """
        self.get_selected_datasets()

        doc = self.db[constants.COL_DATASETS].find_one(
            {'provider_name': self.provider_name, 'dataset_code': dataset_code},
            {'dataset_code': 1, 'last_update': 1})

        dataset_settings = self.selected_datasets[dataset_code]
        
        if doc and  doc['last_update'] >= dataset_settings['last_update']:
            comments = "update-date[%s]" % doc['last_update']
            raise errors.RejectUpdatedDataset(provider_name=self.provider_name,
                                              dataset_code=dataset_code,
                                              comments=comments)            

        dataset = Datasets(provider_name=self.provider_name, 
                           dataset_code=dataset_code, 
                           name=dataset_settings["name"], 
                           doc_href=dataset_settings["metadata"].get("doc_href"), 
                           last_update=dataset_settings["last_update"], 
                           fetcher=self)

        dataset.series.data_iterator = EurostatData(dataset)
        
        return dataset.update_database()

# 需要导入模块: from dlstats.fetchers._commons import Datasets [as 别名]
# 或者: from dlstats.fetchers._commons.Datasets import update_database [as 别名]
    def upsert_dataset(self, dataset_code):
        
        start = time.time()
        logger.info("upsert dataset[%s] - START" % (dataset_code))
        
        #TODO: control si existe ou update !!!

        dataset = Datasets(provider_name=self.provider_name, 
                           dataset_code=dataset_code,
                           name=None,
                           doc_href=self.provider.website,
                           last_update=datetime.now(),
                           fetcher=self)
        
        _data = ECB_Data(dataset=dataset)
        dataset.series.data_iterator = _data
        try:
            result = dataset.update_database()
        except:
            raise
        
        _data = None

        end = time.time() - start
        logger.info("upsert dataset[%s] - END - time[%.3f seconds]" % (dataset_code, end))
        
        return result

# 需要导入模块: from dlstats.fetchers._commons import Datasets [as 别名]
# 或者: from dlstats.fetchers._commons.Datasets import update_database [as 别名]
 def upsert_weo_issue(self, url, dataset_code):
     
     settings = DATASETS[dataset_code]
     
     dataset = Datasets(provider_name=self.provider_name, 
                        dataset_code=dataset_code, 
                        name=settings['name'], 
                        doc_href=settings['doc_href'], 
                        fetcher=self)
     
     weo_data = WeoData(dataset, url)
     dataset.last_update = weo_data.release_date        
     dataset.attribute_list.update_entry('flags','e','Estimated')
     dataset.series.data_iterator = weo_data
     try:
         dataset.update_database()
         self.update_metas(dataset_code)
     except Exception as err:
         logger.error(str(err))

# 需要导入模块: from dlstats.fetchers._commons import Datasets [as 别名]
# 或者: from dlstats.fetchers._commons.Datasets import update_database [as 别名]
    def _common_tests(self):

        self._collections_is_empty()

        self.filepath = get_filepath(self.dataset_code)
        self.assertTrue(os.path.exists(self.filepath))

        # provider.update_database
        self.fetcher.provider.update_database()
        provider = self.db[constants.COL_PROVIDERS].find_one({"name": self.fetcher.provider_name})
        self.assertIsNotNone(provider)

        dataset = Datasets(
            provider_name=self.fetcher.provider_name,
            dataset_code=self.dataset_code,
            name=DATASETS[self.dataset_code]["name"],
            last_update=DATASETS[self.dataset_code]["last_update"],
            fetcher=self.fetcher,
        )

        # manual Data for iterator
        fetcher_data = esri.EsriData(dataset, make_url(self), filename=DATASETS[self.dataset_code]["filename"])
        dataset.series.data_iterator = fetcher_data
        dataset.last_update = DATASETS[self.dataset_code]["last_update"]
        dataset.update_database()

        self.dataset = self.db[constants.COL_DATASETS].find_one(
            {"provider_name": self.fetcher.provider_name, "dataset_code": self.dataset_code}
        )

        self.assertIsNotNone(self.dataset)

        dimensions = self.dataset["dimension_list"]
        self.assertEqual(len(dimensions), DATASETS[self.dataset_code]["dimension_count"])
        for c in dimensions["concept"]:
            self.assertIn(c[1], DATASETS["series_names"])

        series = self.db[constants.COL_SERIES].find(
            {"provider_name": self.fetcher.provider_name, "dataset_code": self.dataset_code}
        )

        self.assertEqual(series.count(), DATASETS[self.dataset_code]["series_count"])