Python Downloader.stop方法代码示例

# 需要导入模块: from sunpy.net.download import Downloader [as 别名]
# 或者: from sunpy.net.download.Downloader import stop [as 别名]
def test_download_http():
    items = []
    lck = threading.Lock()

    def wait_for(n, callback):  # pylint: disable=W0613
        def _fun(handler):
            with lck:
                items.append(handler)
                if len(items) == n:
                    callback(items)
        return _fun

    tmp = tempfile.mkdtemp()
    path_fun = partial(default_name, tmp)

    dw = Downloader(1, 1)
    _stop = lambda _: dw.stop()

    timeout = CalledProxy(dw.stop)
    timer = threading.Timer(60, timeout)
    timer.start()

    on_finish = wait_for(3, lambda _: dw.stop())
    dw.download('http://ajax.googleapis.com/ajax/libs/jquery/2.0.0/jquery.min.js', path_fun, on_finish)
    dw.download('http://ajax.googleapis.com/ajax/libs/webfont/1.4.2/webfont.js', path_fun, on_finish)
    dw.download('https://raw.githubusercontent.com/sunpy/sunpy/master/README.rst', path_fun, on_finish)

    dw.wait()
    timer.cancel()

    assert len(items) == 3
    assert not timeout.fired

    for item in items:
        assert os.path.exists(item['path'])

# 需要导入模块: from sunpy.net.download import Downloader [as 别名]
# 或者: from sunpy.net.download.Downloader import stop [as 别名]
def test_path_exception():
    x = threading.Event()
    dw = Downloader(1, 2)
    dw.download("http://google.at", path_fun, errback=wait_for(1, lambda a: x.set()))
    th = threading.Thread(target=dw.wait)
    th.daemon = True
    th.start()
    x.wait(10)
    assert x.isSet()
    dw.stop()

# 需要导入模块: from sunpy.net.download import Downloader [as 别名]
# 或者: from sunpy.net.download.Downloader import stop [as 别名]
def test_download_default_dir():
    _config = sunpy.config

    try:
        tmpdir = tempfile.mkdtemp()

        sunpy.config = MockConfig()
        sunpy.config.add_section(
            "downloads", {"download_dir": tmpdir}
        )

        dw = Downloader(1, 1)
        _stop = lambda _: dw.stop()

        timeout = CalledProxy(dw.stop)
        errback = CalledProxy(_stop)
        dw.download(
            'http://ajax.googleapis.com/ajax/libs/jquery/2.0.0/jquery.min.js',
            callback=_stop,
            errback=errback
        )

        timer = threading.Timer(10, timeout)
        timer.start()
        dw.wait()
        timer.cancel()

        assert not timeout.fired
        assert not errback.fired
        assert os.path.exists(os.path.join(tmpdir, 'jquery.min.js'))
    finally:
        sunpy.config = _config

# 需要导入模块: from sunpy.net.download import Downloader [as 别名]
# 或者: from sunpy.net.download.Downloader import stop [as 别名]
def test_download_dir():
    tmpdir = tempfile.mkdtemp()

    dw = Downloader(1, 1)
    _stop = lambda _: dw.stop()
    timeout = CalledProxy(dw.stop)
    errback = CalledProxy(_stop)

    dw.download(
        "http://ajax.googleapis.com/ajax/libs/jquery/2.0.0/jquery.min.js", tmpdir, callback=_stop, errback=errback
    )

    timer = threading.Timer(10, timeout)
    timer.start()
    dw.wait()
    timer.cancel()
    assert not timeout.fired
    assert not errback.fired
    assert os.path.exists(os.path.join(tmpdir, "jquery.min.js"))

# 需要导入模块: from sunpy.net.download import Downloader [as 别名]
# 或者: from sunpy.net.download.Downloader import stop [as 别名]
    def get_request(self, requestIDs, path=None, overwrite=False, progress=True,
                    max_conn=5, downloader=None, results=None):
        """
        Query JSOC to see if request_id is ready for download.

        If the request is ready for download, download it.

        Parameters
        ----------
        requestIDs : list or string
            One or many requestID strings

        path : string
            Path to save data to, defaults to SunPy download dir

        overwrite : bool
            Replace files with the same name if True

        progress : bool
            Print progress info to terminal

        max_conns : int
            Maximum number of download connections.

        downloader : `sunpy.download.Downloader` instance
            A Custom downloader to use

        results: Results instance
            A Results manager to use.

        Returns
        -------
        res: Results
            A Results instance or None if no URLs to download
        """

        # Convert IDs to a list if not already

        if not isiterable(requestIDs) or isinstance(requestIDs, six.string_types):
            requestIDs = [requestIDs]

        if path is None:
            path = config.get('downloads', 'download_dir')
        path = os.path.expanduser(path)

        if downloader is None:
            downloader = Downloader(max_conn=max_conn, max_total=max_conn)

        # A Results object tracks the number of downloads requested and the
        # number that have been completed.
        if results is None:
            results = Results(lambda _: downloader.stop())

        urls = []
        for request_id in requestIDs:
            u = self._request_status(request_id)

            if u.status_code == 200 and u.json()['status'] == '0':
                for ar in u.json()['data']:
                    is_file = os.path.isfile(os.path.join(path, ar['filename']))
                    if overwrite or not is_file:
                        url_dir = BASE_DL_URL + u.json()['dir'] + '/'
                        urls.append(urllib.parse.urljoin(url_dir, ar['filename']))

                    else:
                        print_message = "Skipping download of file {} as it " \
                                        "has already been downloaded"
                        print(print_message.format(ar['filename']))
                        # Add the file on disk to the output
                        results.map_.update({ar['filename']:
                                            {'path': os.path.join(path, ar['filename'])}})

                if progress:
                    print_message = "{0} URLs found for download. Totalling {1}MB"
                    print(print_message.format(len(urls), u.json()['size']))

            else:
                if progress:
                    self.check_request(request_id)

        if urls:
            for url in urls:
                downloader.download(url, callback=results.require([url]),
                                    errback=lambda x: print(x), path=path)

        else:
            # Make Results think it has finished.
            results.require([])
            results.poke()

        return results

# 需要导入模块: from sunpy.net.download import Downloader [as 别名]
# 或者: from sunpy.net.download.Downloader import stop [as 别名]
    def get_request(self, requests, path=None, overwrite=False, progress=True,
                    max_conn=5, downloader=None, results=None):
        """
        Query JSOC to see if the request(s) is ready for download.

        If the request is ready for download, it will then download it.

        Parameters
        ----------
        requests : `~drms.ExportRequest`, `str`, `list`
            `~drms.ExportRequest` objects or `str` request IDs or lists
            returned by `~sunpy.net.jsoc.jsoc.JSOCClient.request_data`.

        path : `str`
            Path to save data to, defaults to SunPy download dir.

        overwrite : `bool`
            Replace files with the same name if True.

        progress : `bool`
            Print progress info to terminal.

        max_conns : `int`
            Maximum number of download connections.

        downloader : `~sunpy.net.download.Downloader`
            A Custom downloader to use

        results: `~sunpy.net.download.Results`
            A `~sunpy.net.download.Results` manager to use.

        Returns
        -------
        res: `~sunpy.net.download.Results`
            A `~sunpy.net.download.Results` instance or `None` if no URLs to download

        """
        c = drms.Client()

        # Convert Responses to a list if not already
        if isinstance(requests, six.string_types) or not isiterable(requests):
            requests = [requests]

        # Ensure all the requests are drms ExportRequest objects
        for i, request in enumerate(requests):
            if isinstance(request, six.string_types):
                r = c.export_from_id(request)
                requests[i] = r

        # We only download if all are finished
        if not all([r.has_succeeded() for r in requests]):
            raise NotExportedError("Can not download as not all the requests"
                                   "have been exported for download yet.")

        # Ensure path has a {file} in it
        if path is None:
            default_dir = config.get("downloads", "download_dir")
            path = os.path.join(default_dir, '{file}')
        elif isinstance(path, six.string_types) and '{file}' not in path:
            path = os.path.join(path, '{file}')

        paths = []
        for request in requests:
            for filename in request.data['filename']:
                # Ensure we don't duplicate the file extension
                ext = os.path.splitext(filename)[1]
                if path.endswith(ext):
                    fname = path.strip(ext)
                else:
                    fname = path
                fname = fname.format(file=filename)
                fname = os.path.expanduser(fname)
                fname = partial(simple_path, fname)
                paths.append(fname)

        if downloader is None:
            downloader = Downloader(max_conn=max_conn, max_total=max_conn)

        # A Results object tracks the number of downloads requested and the
        # number that have been completed.
        if results is None:
            results = Results(lambda _: downloader.stop(),
                              done=lambda maps: [v['path'] for v in maps.values()])

        urls = []
        for request in requests:

            if request.status == 0:
                for index, data in request.data.iterrows():
                    is_file = os.path.isfile(paths[index].args[0])
                    if overwrite or not is_file:
                        url_dir = request.request_url + '/'
                        urls.append(urllib.parse.urljoin(url_dir, data['filename']))

                    if not overwrite and is_file:
                        print_message = "Skipping download of file {} as it " \
                                        "has already been downloaded. " \
                                        "If you want to redownload the data, "\
                                        "please set overwrite to True"
                        print(print_message.format(data['filename']))
#.........这里部分代码省略.........