Python download.Downloader类代码示例

def test_path_exception():
    x = threading.Event()
    dw = Downloader(1, 2)
    dw.download("http://google.at", path_fun, errback=wait_for(1, lambda a: x.set()))
    threading.Thread(target=dw.reactor.run).start()
    assert x.wait(10)
    dw.reactor.stop()

def test_path_exception():
    x = threading.Event()
    dw = Downloader(1, 2)
    dw.download("http://google.at", path_fun, errback=wait_for(1, lambda a: x.set()))
    th = threading.Thread(target=dw.wait)
    th.daemon = True
    th.start()
    x.wait(10)
    assert x.isSet()
    dw.stop()

    def fetch(self, qres, path=None, error_callback=None, **kwargs):
        """
        Download a set of results.

        Parameters
        ----------
        qres : `~sunpy.net.dataretriever.QueryResponse`
            Results to download.

        path : string or pathlib.Path
            Path to the download directory

        error_callback : Function
            Callback function for error during downloads

        Returns
        -------
        Results Object
        """
        # Check for type of path
        if path is not None:
            if HAS_PATHLIB and isinstance(path, pathlib.Path):
                path = str(path.absolute())
            elif not isinstance(path, six.string_types):
                err = "path should be either 'pathlib.Path' or 'str'. "\
                    "Got '{}'.".format(type(path))
                raise TypeError(err)

        urls = [qrblock.url for qrblock in qres]

        filenames = [url.split('/')[-1] for url in urls]

        paths = self._get_full_filenames(qres, filenames, path)

        res = Results(lambda x: None, 0, lambda map_: self._link(map_))

        dobj = Downloader(max_conn=len(urls), max_total=len(urls))

        # We cast to list here in list(zip... to force execution of
        # res.require([x]) at the start of the loop.
        for aurl, ncall, fname in list(zip(urls, map(lambda x: res.require([x]),
                                                     urls), paths)):
            dobj.download(aurl, fname, ncall, error_callback)

        return res

def test_download_default_dir():
    _config = sunpy.config

    try:
        tmpdir = tempfile.mkdtemp()

        sunpy.config = MockConfig()
        sunpy.config.add_section(
            "downloads", {"download_dir": tmpdir}
        )

        dw = Downloader(1, 1)
        _stop = lambda _: dw.stop()

        timeout = CalledProxy(dw.stop)
        errback = CalledProxy(_stop)
        dw.download(
            'http://ajax.googleapis.com/ajax/libs/jquery/2.0.0/jquery.min.js',
            callback=_stop,
            errback=errback
        )

        timer = threading.Timer(10, timeout)
        timer.start()
        dw.wait()
        timer.cancel()

        assert not timeout.fired
        assert not errback.fired
        assert os.path.exists(os.path.join(tmpdir, 'jquery.min.js'))
    finally:
        sunpy.config = _config

def test_download_http():
    items = []
    lck = threading.Lock()

    def wait_for(n, callback):  # pylint: disable=W0613
        def _fun(handler):
            with lck:
                items.append(handler)
                if len(items) == n:
                    callback(items)

        return _fun

    tmp = tempfile.mkdtemp()
    path_fun = partial(default_name, tmp)

    dw = Downloader(1, 1)

    timeout = CalledProxy(dw.stop)
    timer = threading.Timer(60, timeout)
    timer.start()

    on_finish = wait_for(3, lambda _: dw.stop())
    dw.download("http://ajax.googleapis.com/ajax/libs/jquery/2.0.0/jquery.min.js", path_fun, on_finish)
    dw.download("http://ajax.googleapis.com/ajax/libs/webfont/1.4.2/webfont.js", path_fun, on_finish)
    dw.download("https://raw.github.com/sunpy/sunpy/master/INSTALL.txt", path_fun, on_finish)
    # dw.download('ftp://speedtest.inode.at/speedtest-100mb', path_fun, on_finish)

    dw.wait()
    timer.cancel()

    assert len(items) == 3
    assert not timeout.fired

    for item in items:
        assert os.path.exists(item["path"])

def test_download_dir():
    tmpdir = tempfile.mkdtemp()

    dw = Downloader(1, 1)
    _stop = lambda _: dw.stop()
    timeout = CalledProxy(dw.stop)
    errback = CalledProxy(_stop)

    dw.download(
        "http://ajax.googleapis.com/ajax/libs/jquery/2.0.0/jquery.min.js", tmpdir, callback=_stop, errback=errback
    )

    timer = threading.Timer(10, timeout)
    timer.start()
    dw.wait()
    timer.cancel()
    assert not timeout.fired
    assert not errback.fired
    assert os.path.exists(os.path.join(tmpdir, "jquery.min.js"))

    def fetch(self, qres, path=None, error_callback=None, **kwargs):
        """
        Download a set of results.

        Parameters
        ----------
        qres : `~sunpy.net.dataretriever.QueryResponse`
            Results to download.

        Returns
        -------
        Results Object
        """

        urls = [qrblock.url for qrblock in qres]

        filenames = []
        local_filenames = []

        for i, [url, qre] in enumerate(zip(urls, qres)):
            name = url.split('/')[-1]

            # temporary fix !!! coz All QRBs have same start_time values
            day = Time(qre.time.start.strftime('%Y-%m-%d')) + TimeDelta(i*u.day)

            if name not in filenames:
                filenames.append(name)

            if name.endswith('.gz'):
                local_filenames.append('{}SRS.txt'.format(day.strftime('%Y%m%d')))
            else:
                local_filenames.append(name)

        # Files to be actually downloaded
        paths = self._get_full_filenames(qres, filenames, path)

        # Those files that will be present after get returns
        local_paths = self._get_full_filenames(qres, local_filenames, path)

        res = Results(lambda x: None, 0, lambda map_: self._link(map_))

        # remove duplicate urls. This will make paths and urls to have same number of elements.
        # OrderedDict is required to maintain ordering because it will be zipped with paths later

        urls = list(OrderedDict.fromkeys(urls))

        dobj = Downloader(max_conn=len(urls), max_total=len(urls))

        # We cast to list here in list(zip... to force execution of
        # res.require([x]) at the start of the loop.
        for aurl, ncall, fname in list(zip(urls, map(lambda x: res.require([x]),
                                                     urls), paths)):
            dobj.download(aurl, fname, ncall, error_callback)

        res.wait()

        res2 = Results(lambda x: None, 0)

        for fname, srs_filename in zip(local_paths, local_filenames):

            fname = fname.args[0]
            name = fname.split('/')[-1]

            past_year = False
            for i, fname2 in enumerate(paths):

                fname2 = fname2.args[0]

                if fname2.endswith('.txt'):
                    continue

                year = fname2.split('/')[-1]
                year = year.split('_SRS')[0]

                if year in name:
                    TarFile = tarfile.open(fname2)
                    filepath = fname.rpartition('/')[0]
                    member = TarFile.getmember('SRS/' + srs_filename)
                    member.name = name
                    TarFile.extract(member, path=filepath)
                    TarFile.close()

                    callback = res2.require([fname])
                    callback({'path': fname})

                    past_year = True
                    break

            if past_year is False:
                callback = res2.require([fname])
                callback({'path': fname})

        return res2

    def get(self, requestIDs, path=None, overwrite=False, progress=True,
            max_conn=5, downloader=None, results=None):
        """
        Query JSOC to see if request_id is ready for download.

        If the request is ready for download download it.

        Parameters
        ----------
        requestIDs: list or string
            One or many requestID strings

        path: string
            Path to save data to, defaults to SunPy download dir

        overwrite: bool
            Replace files with the same name if True

        progress: bool
            Print progress info to terminal

        max_conns: int
            Maximum number of download connections.

        downloader: sunpy.download.Downloder instance
            A Custom downloader to use

        results: Results instance
            A Results manager to use.

        Returns
        -------
        res: Results
            A Results instance or None if no URLs to download
        """

        # Convert IDs to a list if not already
        if not astropy.utils.misc.isiterable(requestIDs) or isinstance(requestIDs, basestring):
            requestIDs = [requestIDs]

        if path is None:
            path = config.get('downloads','download_dir')

        if downloader is None:
            downloader = Downloader(max_conn=max_conn, max_total=max_conn)

        # A Results object tracks the number of downloads requested and the
        # number that have been completed.
        if results is None:
            results = Results(lambda x: None)

        urls = []
        for request_id in requestIDs:
            u = self._request_status(request_id)

            if u.status_code == 200 and u.json()['status'] == '0':
                for ar in u.json()['data']:
                    if overwrite or not os.path.isfile(os.path.join(path, ar['filename'])):
                        urls.append(urlparse.urljoin(BASE_DL_URL + u.json()['dir']+'/', ar['filename']))
                if progress:
                    print("{0} URLs found for Download. Totalling {1}MB".format(len(urls), u.json()['size']))

            else:
                if progress:
                    self.check_request(request_id)

        if urls:
            for url, rcall in list(zip(urls, list(map(lambda x: results.require([x]), urls)))):
                downloader.download(url, callback=rcall, path=path)

        else:
            #Make Results think it has finished.
            results.require([])

        results.poke()
        return results

    def get_request(self, requestIDs, path=None, overwrite=False, progress=True,
                    max_conn=5, downloader=None, results=None):
        """
        Query JSOC to see if request_id is ready for download.

        If the request is ready for download, download it.

        Parameters
        ----------
        requestIDs : list or string
            One or many requestID strings

        path : string
            Path to save data to, defaults to SunPy download dir

        overwrite : bool
            Replace files with the same name if True

        progress : bool
            Print progress info to terminal

        max_conns : int
            Maximum number of download connections.

        downloader : `sunpy.download.Downloader` instance
            A Custom downloader to use

        results: Results instance
            A Results manager to use.

        Returns
        -------
        res: Results
            A Results instance or None if no URLs to download
        """

        # Convert IDs to a list if not already

        if not isiterable(requestIDs) or isinstance(requestIDs, six.string_types):
            requestIDs = [requestIDs]

        if path is None:
            path = config.get('downloads', 'download_dir')
        path = os.path.expanduser(path)

        if downloader is None:
            downloader = Downloader(max_conn=max_conn, max_total=max_conn)

        # A Results object tracks the number of downloads requested and the
        # number that have been completed.
        if results is None:
            results = Results(lambda _: downloader.stop())

        urls = []
        for request_id in requestIDs:
            u = self._request_status(request_id)

            if u.status_code == 200 and u.json()['status'] == '0':
                for ar in u.json()['data']:
                    is_file = os.path.isfile(os.path.join(path, ar['filename']))
                    if overwrite or not is_file:
                        url_dir = BASE_DL_URL + u.json()['dir'] + '/'
                        urls.append(urllib.parse.urljoin(url_dir, ar['filename']))

                    else:
                        print_message = "Skipping download of file {} as it " \
                                        "has already been downloaded"
                        print(print_message.format(ar['filename']))
                        # Add the file on disk to the output
                        results.map_.update({ar['filename']:
                                            {'path': os.path.join(path, ar['filename'])}})

                if progress:
                    print_message = "{0} URLs found for download. Totalling {1}MB"
                    print(print_message.format(len(urls), u.json()['size']))

            else:
                if progress:
                    self.check_request(request_id)

        if urls:
            for url in urls:
                downloader.download(url, callback=results.require([url]),
                                    errback=lambda x: print(x), path=path)

        else:
            # Make Results think it has finished.
            results.require([])
            results.poke()

        return results

    def get_request(self, requests, path=None, overwrite=False, progress=True,
                    max_conn=5, downloader=None, results=None):
        """
        Query JSOC to see if the request(s) is ready for download.

        If the request is ready for download, it will then download it.

        Parameters
        ----------
        requests : `~drms.ExportRequest`, `str`, `list`
            `~drms.ExportRequest` objects or `str` request IDs or lists
            returned by `~sunpy.net.jsoc.jsoc.JSOCClient.request_data`.

        path : `str`
            Path to save data to, defaults to SunPy download dir.

        overwrite : `bool`
            Replace files with the same name if True.

        progress : `bool`
            Print progress info to terminal.

        max_conns : `int`
            Maximum number of download connections.

        downloader : `~sunpy.net.download.Downloader`
            A Custom downloader to use

        results: `~sunpy.net.download.Results`
            A `~sunpy.net.download.Results` manager to use.

        Returns
        -------
        res: `~sunpy.net.download.Results`
            A `~sunpy.net.download.Results` instance or `None` if no URLs to download

        """
        c = drms.Client()

        # Convert Responses to a list if not already
        if isinstance(requests, six.string_types) or not isiterable(requests):
            requests = [requests]

        # Ensure all the requests are drms ExportRequest objects
        for i, request in enumerate(requests):
            if isinstance(request, six.string_types):
                r = c.export_from_id(request)
                requests[i] = r

        # We only download if all are finished
        if not all([r.has_succeeded() for r in requests]):
            raise NotExportedError("Can not download as not all the requests"
                                   "have been exported for download yet.")

        # Ensure path has a {file} in it
        if path is None:
            default_dir = config.get("downloads", "download_dir")
            path = os.path.join(default_dir, '{file}')
        elif isinstance(path, six.string_types) and '{file}' not in path:
            path = os.path.join(path, '{file}')

        paths = []
        for request in requests:
            for filename in request.data['filename']:
                # Ensure we don't duplicate the file extension
                ext = os.path.splitext(filename)[1]
                if path.endswith(ext):
                    fname = path.strip(ext)
                else:
                    fname = path
                fname = fname.format(file=filename)
                fname = os.path.expanduser(fname)
                fname = partial(simple_path, fname)
                paths.append(fname)

        if downloader is None:
            downloader = Downloader(max_conn=max_conn, max_total=max_conn)

        # A Results object tracks the number of downloads requested and the
        # number that have been completed.
        if results is None:
            results = Results(lambda _: downloader.stop(),
                              done=lambda maps: [v['path'] for v in maps.values()])

        urls = []
        for request in requests:

            if request.status == 0:
                for index, data in request.data.iterrows():
                    is_file = os.path.isfile(paths[index].args[0])
                    if overwrite or not is_file:
                        url_dir = request.request_url + '/'
                        urls.append(urllib.parse.urljoin(url_dir, data['filename']))

                    if not overwrite and is_file:
                        print_message = "Skipping download of file {} as it " \
                                        "has already been downloaded. " \
                                        "If you want to redownload the data, "\
                                        "please set overwrite to True"
                        print(print_message.format(data['filename']))
#.........这里部分代码省略.........