Python Downloader.get_filepath方法代码示例

# 需要导入模块: from dlstats.utils import Downloader [as 别名]
# 或者: from dlstats.utils.Downloader import get_filepath [as 别名]
    def _load_structure(self, force=False):
        
        if self._dataflows and not force:
            return

        self.xml_sdmx = XMLSDMX(agencyID=self.provider_name)
        
        self.xml_dsd = XMLStructure(provider_name=self.provider_name,
                                    sdmx_client=self.xml_sdmx)       
        
        url = "http://www.bdm.insee.fr/series/sdmx/dataflow/%s" % self.provider_name
        download = Downloader(url=url, 
                              filename="dataflow.xml",
                              store_filepath=self.store_path,
                              headers=SDMX_METADATA_HEADERS,
                              use_existing_file=self.use_existing_file)
        filepath = download.get_filepath()
        self.for_delete.append(filepath)
        self.xml_dsd.process(filepath)
        self._dataflows = self.xml_dsd.dataflows

        url = "http://www.bdm.insee.fr/series/sdmx/categoryscheme/%s" % self.provider_name
        download = Downloader(url=url, 
                              filename="categoryscheme.xml",
                              store_filepath=self.store_path,
                              headers=SDMX_METADATA_HEADERS,
                              use_existing_file=self.use_existing_file)
        filepath = download.get_filepath()
        self.for_delete.append(filepath)
        self.xml_dsd.process(filepath)
        self._categoryschemes = self.xml_dsd.categories

        url = "http://www.bdm.insee.fr/series/sdmx/categorisation/%s" % self.provider_name
        download = Downloader(url=url, 
                              filename="categorisation.xml",
                              store_filepath=self.store_path,
                              headers=SDMX_METADATA_HEADERS,
                              use_existing_file=self.use_existing_file)
        filepath = download.get_filepath()
        self.for_delete.append(filepath)
        self.xml_dsd.process(filepath)
        self._categorisations = self.xml_dsd.categorisations
        
        url = "http://www.bdm.insee.fr/series/sdmx/conceptscheme/%s" % self.provider_name
        download = Downloader(url=url, 
                              filename="conceptscheme.xml",
                              store_filepath=self.store_path,
                              headers=SDMX_METADATA_HEADERS,
                              use_existing_file=self.use_existing_file)
        filepath = download.get_filepath()
        self.for_delete.append(filepath)
        self.xml_dsd.process(filepath)
        self._concepts = self.xml_dsd.concepts

# 需要导入模块: from dlstats.utils import Downloader [as 别名]
# 或者: from dlstats.utils.Downloader import get_filepath [as 别名]
    def _load(self):

        url = "http://sdw-wsrest.ecb.int/service/dataflow/ECB/%s" % self.dataset_code
        download = Downloader(url=url, 
                              filename="dataflow-%s.xml" % self.dataset_code,
                              headers=SDMX_METADATA_HEADERS)
        
        self.xml_dsd.process(download.get_filepath())
        self.dsd_id = self.xml_dsd.dsd_id
        
        if not self.dsd_id:
            msg = "DSD ID not found for provider[%s] - dataset[%s]" % (self.provider_name, 
                                                                       self.dataset_code)
            raise Exception(msg)
        
        url = "http://sdw-wsrest.ecb.int/service/datastructure/ECB/%s?references=children" % self.dsd_id
        download = Downloader(url=url, 
                              filename="dsd-%s.xml" % self.dataset_code,
                              headers=SDMX_METADATA_HEADERS)
        self.xml_dsd.process(download.get_filepath())
        
        self.dataset.name = self.xml_dsd.dataset_name
        
        dimensions = OrderedDict()
        for key, item in self.xml_dsd.dimensions.items():
            dimensions[key] = item["dimensions"]
        self.dimension_list.set_dict(dimensions)
        
        attributes = OrderedDict()
        for key, item in self.xml_dsd.attributes.items():
            attributes[key] = item["values"]
        self.attribute_list.set_dict(attributes)
        
        url = "http://sdw-wsrest.ecb.int/service/data/%s" % self.dataset_code
        download = Downloader(url=url, 
                              filename="data-%s.xml" % self.dataset_code,
                              headers=SDMX_DATA_HEADERS)

        self.xml_data = XMLData(provider_name=self.provider_name,
                                dataset_code=self.dataset_code,
                                dimension_keys=self.xml_dsd.dimension_keys)
        
        
        #TODO: response and exception
        try:
            filepath, response = download.get_filepath_and_response()        
        except requests.exceptions.HTTPError as err:
            logger.critical("AUTRE ERREUR HTTP : %s" % err.response.status_code)
            raise
            
        self.rows = self.xml_data.process(filepath)

# 需要导入模块: from dlstats.utils import Downloader [as 别名]
# 或者: from dlstats.utils.Downloader import get_filepath [as 别名]
    def _load_structure_datatree(self, force=False):

        if self._categoryschemes and self._categorisations and not force:
            return

        self._load_structure_dataflows(force)

        url = "http://www.bdm.insee.fr/series/sdmx/categoryscheme/%s" % self.provider_name
        """
        if self.refresh_meta is False:
            self._categoryschemes = self._structure_get("categoryschemes")
            if self._categoryschemes:
                logger.info("load structure [categoryschemes] from metadata for url[%s]" % url)
        """
        if not self._categoryschemes:
            download = Downloader(url=url,
                                  filename="categoryscheme.xml",
                                  store_filepath=self.store_path,
                                  headers=SDMX_METADATA_HEADERS,
                                  use_existing_file=self.use_existing_file,
                                  client=self.requests_client)
            filepath = download.get_filepath()
            self.for_delete.append(filepath)
            self.xml_dsd.process(filepath)
            self._categoryschemes = self.xml_dsd.categories
            #self._structure_put("categoryschemes", url, **self._categoryschemes)

        url = "http://www.bdm.insee.fr/series/sdmx/categorisation/%s" % self.provider_name
        """
        if self.refresh_meta is False:
            self._categorisations = self._structure_get("categorisation")
            if self._categorisations:
                self._categorisations_categories = self._structure_get("categorisations_categories")
                logger.info("load structure [categorisation] from metadata for url[%s]" % url)
        """

        if not self._categorisations:
            download = Downloader(url=url,
                                  filename="categorisation.xml",
                                  store_filepath=self.store_path,
                                  headers=SDMX_METADATA_HEADERS,
                                  use_existing_file=self.use_existing_file,
                                  client=self.requests_client)
            filepath = download.get_filepath()
            self.for_delete.append(filepath)
            self.xml_dsd.process(filepath)
            self._categorisations = self.xml_dsd.categorisations
            self._categorisations_categories = self.xml_dsd.categorisations_categories

# 需要导入模块: from dlstats.utils import Downloader [as 别名]
# 或者: from dlstats.utils.Downloader import get_filepath [as 别名]
 def _load_datas(self, datas=None):
     
     kwargs = {}
     
     if not datas:
         # TODO: timeout, replace
         download = Downloader(url=self.url,
                               store_filepath=self.store_path, 
                               filename=self.filename,
                               use_existing_file=self.fetcher.use_existing_file)
         
         zip_filepath = download.get_filepath()
         self.fetcher.for_delete.append(zip_filepath)
         filepath = extract_zip_file(zip_filepath)
         self.fetcher.for_delete.append(zip_filepath)
         
         kwargs['filepath'] = filepath
     else:
         kwargs['fileobj'] = io.StringIO(datas, newline="\n")
     
     kwargs['date_format'] = "%a %b %d %H:%M:%S %Z %Y"
     kwargs['headers_line'] = DATASETS[self.dataset.dataset_code]['lines']['headers']
     self._file, self._rows, self.headers, self.release_date, self.dimension_keys, self.periods = local_read_csv(**kwargs)
     
     self.dataset.dimension_keys = self.dimension_keys
     
     self.dataset.last_update = self.release_date
     
     self.start_date = get_ordinal_from_period(self.periods[0], freq=self.frequency)
     self.end_date = get_ordinal_from_period(self.periods[-1], freq=self.frequency)

# 需要导入模块: from dlstats.utils import Downloader [as 别名]
# 或者: from dlstats.utils.Downloader import get_filepath [as 别名]
    def weo_urls(self):
        download = Downloader(url='http://www.imf.org/external/ns/cs.aspx?id=28',
                              filename="weo.html",
                              store_filepath=self.store_path)
        
        filepath = download.get_filepath()
        with open(filepath, 'rb') as fp:
            webpage = fp.read()
        
        self.fetcher.for_delete.append(filepath)
            
        #TODO: replace by beautifoulsoup ?
        html = etree.HTML(webpage)
        hrefs = html.xpath("//div[@id = 'content-main']/h4/a['href']")
        links = [href.values() for href in hrefs]
        
        #The last links of the WEO webpage lead to data we dont want to pull.
        links = links[:-16]
        #These are other links we don't want.
        links.pop(-8)
        links.pop(-10)
        links = [link[0][:-10]+'download.aspx' for link in links]

        output = []
    
        for link in links:
            webpage = requests.get(link)
            html = etree.HTML(webpage.text)
            final_link = html.xpath("//div[@id = 'content']//table//a['href']")
            output.append(link[:-13]+final_link[0].values()[0])
            
        # we need to handle the issue in chronological order
        return sorted(output)

# 需要导入模块: from dlstats.utils import Downloader [as 别名]
# 或者: from dlstats.utils.Downloader import get_filepath [as 别名]
 def _load_datas(self):
     
     store_filepath = self.get_store_path()
     # TODO: timeout, replace
     download = Downloader(url=self.dataset_url, filename=self.filename, store_filepath=store_filepath)
         
     return(download.get_filepath())

# 需要导入模块: from dlstats.utils import Downloader [as 别名]
# 或者: from dlstats.utils.Downloader import get_filepath [as 别名]
    def _load(self):

        download = Downloader(
            url=self.dataset_url,
            filename="data-%s.zip" % self.dataset_code,
            store_filepath=self.store_path,
            use_existing_file=self.fetcher.use_existing_file,
        )

        filepaths = extract_zip_file(download.get_filepath())
        dsd_fp = filepaths[self.dataset_code + ".dsd.xml"]
        data_fp = filepaths[self.dataset_code + ".sdmx.xml"]

        self.fetcher.for_delete.append(dsd_fp)
        self.fetcher.for_delete.append(data_fp)

        self.xml_dsd.process(dsd_fp)
        self._set_dataset()

        self.xml_data = XMLData(
            provider_name=self.provider_name,
            dataset_code=self.dataset_code,
            xml_dsd=self.xml_dsd,
            dsd_id=self.dataset_code,
            # TODO: frequencies_supported=FREQUENCIES_SUPPORTED
        )
        self.rows = self.xml_data.process(data_fp)

# 需要导入模块: from dlstats.utils import Downloader [as 别名]
# 或者: from dlstats.utils.Downloader import get_filepath [as 别名]
    def _load_structure_dataflows(self, force=False):

        if self._dataflows and not force:
            return

        self.provider_verify()

        url = "http://www.bdm.insee.fr/series/sdmx/dataflow/%s" % self.provider_name

        if self.refresh_meta is False:
            self._dataflows = self._structure_get("dataflows")

            if self._dataflows:
                self.xml_dsd.dataflows = self._dataflows
                logger.info("load structure [dataflows] from metadata for url[%s]" % url)
                return

        download = Downloader(url=url,
                              filename="dataflow.xml",
                              store_filepath=self.store_path,
                              headers=SDMX_METADATA_HEADERS,
                              use_existing_file=self.use_existing_file,
                              client=self.requests_client)
        filepath = download.get_filepath()
        self.for_delete.append(filepath)
        self.xml_dsd.process(filepath)
        self._dataflows = self.xml_dsd.dataflows

        self._structure_put("dataflows", url, **self._dataflows)

# 需要导入模块: from dlstats.utils import Downloader [as 别名]
# 或者: from dlstats.utils.Downloader import get_filepath [as 别名]
 def _load_xls(self):
     url_xls = make_xls_url(self.dataset_code)
     download = Downloader(url=url_xls, 
                       filename=self.dataset_code + '_info.xls',
                       store_filepath=self.get_store_path(),
                       use_existing_file=self.fetcher.use_existing_file)
     filepath = download.get_filepath()
     return filepath

# 需要导入模块: from dlstats.utils import Downloader [as 别名]
# 或者: from dlstats.utils.Downloader import get_filepath [as 别名]
 def _load_datas(self):
     # TODO: timeout, replace
     download = Downloader(url=self.dataset_url, 
                           filename=self.dataset_code,
                           store_filepath=self.store_path,
                           use_existing_file=self.fetcher.use_existing_file)
     filepath = download.get_filepath()
     self.fetcher.for_delete.append(filepath)
     return filepath

# 需要导入模块: from dlstats.utils import Downloader [as 别名]
# 或者: from dlstats.utils.Downloader import get_filepath [as 别名]
    def _load_datas(self):

        store_filepath = self.get_store_path()
        download = Downloader(url=self.dataset_url, 
                              filename=self.filename, 
                              store_filepath=store_filepath)

        '''Return 2 filepath (dsd and data)'''    
        return (extract_zip_file(download.get_filepath()))

# 需要导入模块: from dlstats.utils import Downloader [as 别名]
# 或者: from dlstats.utils.Downloader import get_filepath [as 别名]
    def _get_agenda(self):
        download = Downloader(url=AGENDA['url'],
                              filename=AGENDA['filename'],
                              store_filepath=self.store_path)
        filepath = download.get_filepath()        

        with open(filepath, 'rb') as fp:
            content = fp.read()
            self.for_delete.append(filepath)
            return content

# 需要导入模块: from dlstats.utils import Downloader [as 别名]
# 或者: from dlstats.utils.Downloader import get_filepath [as 别名]
 def _load_datas(self):
     filepath = list()
     if self.dataset_url:
         download = Downloader(url=self.dataset_url, 
                               filename=self.dataset_code,
                               store_filepath=self.get_store_path(),
                               use_existing_file=self.fetcher.use_existing_file)
         filepath.append(download.get_filepath())
     else:
         _filter_name, _filter_codes = get_filter(self.dataset_code)
         for code in _filter_codes:
             url = "http://webstat.banque-france.fr/en/export.do?node=DATASETS_%s&%s=%s&exportType=sdmx" % (self.dataset_code, _filter_name, code)              
             name = self.dataset_code + "_%s" % (code) 
             # print(name)
             download = Downloader(url=url, 
                                   filename=name,
                                   store_filepath=self.get_store_path(),
                                   use_existing_file=self.fetcher.use_existing_file)
             filepath.append(download.get_filepath())
     return filepath 

# 需要导入模块: from dlstats.utils import Downloader [as 别名]
# 或者: from dlstats.utils.Downloader import get_filepath [as 别名]
 def _load_dsd(self):
     url = self._get_url_dsd()
     download = Downloader(store_filepath=self.store_path,
                           url=url, 
                           filename="dsd-%s.xml" % self.dataset_code,
                           use_existing_file=self.fetcher.use_existing_file,
                           client=self.fetcher.requests_client)
     filepath = download.get_filepath()
     self.fetcher.for_delete.append(filepath)
     
     self.xml_dsd.process(filepath)
     self._set_dataset()

# 需要导入模块: from dlstats.utils import Downloader [as 别名]
# 或者: from dlstats.utils.Downloader import get_filepath [as 别名]
    def _load(self):

        download = Downloader(url=self.url, 
                              filename="data-%s.xml" % self.dataset_code,
                              #headers=SDMX_DATA_HEADERS        
                              )
        data_fp, dsd_fp = (extract_zip_file(download.get_filepath()))

        self.xml_data = XMLData(provider_name=self.provider_name,
                                dataset_code=self.dataset_code,
                                #dimension_keys=self.xml_dsd.dimension_keys
                                )
        
        self.rows = self.xml_data.process(data_fp)