Python data.download_file方法代码示例

# 需要导入模块: from astropy.utils import data [as 别名]
# 或者: from astropy.utils.data import download_file [as 别名]
def test_effective_focal_length():
    test_file_url = (
        "https://github.com/cta-observatory/pyeventio/raw/master/tests"
        "/resources/prod4_pixelsettings_v3.gz"
    )
    test_file = download_file(test_file_url)

    focal_length_nominal = 0
    focal_length_effective = 0

    with SimTelEventSource(
        input_url=test_file, focal_length_choice="nominal"
    ) as source:
        subarray = source.subarray
        focal_length_nominal = subarray.tel[1].optics.equivalent_focal_length

    with SimTelEventSource(
        input_url=test_file, focal_length_choice="effective"
    ) as source:
        subarray = source.subarray
        focal_length_effective = subarray.tel[1].optics.equivalent_focal_length

    assert focal_length_nominal > 0
    assert focal_length_effective > 0
    assert focal_length_nominal != focal_length_effective 

# 需要导入模块: from astropy.utils import data [as 别名]
# 或者: from astropy.utils.data import download_file [as 别名]
def test_url_or_file_ephemeris(time):
    # URL for ephemeris de432s used for testing:
    url = 'http://naif.jpl.nasa.gov/pub/naif/generic_kernels/spk/planets/de432s.bsp'

    # Pass the ephemeris directly as a URL.
    coord_by_url = get_body('earth', time, ephemeris=url)

    # Translate the URL to the cached location on the filesystem.
    # Since we just used the url above, it should already have been downloaded.
    filepath = download_file(url, cache=True)

    # Get the coordinates using the file path directly:
    coord_by_filepath = get_body('earth', time, ephemeris=filepath)

    # Using the URL or filepath should give exactly the same results:
    assert_quantity_allclose(coord_by_url.ra, coord_by_filepath.ra)
    assert_quantity_allclose(coord_by_url.dec, coord_by_filepath.dec)
    assert_quantity_allclose(coord_by_url.distance, coord_by_filepath.distance) 

# 需要导入模块: from astropy.utils import data [as 别名]
# 或者: from astropy.utils.data import download_file [as 别名]
def fake_readonly_cache(tmpdir, valid_urls, monkeypatch):
    def no_mkdir(p):
        raise PermissionError("os.mkdir monkeypatched out")

    with TemporaryDirectory(dir=tmpdir) as d:
        # other fixtures use the same tmpdir so we need a subdirectory
        # to make into the cache
        d = pathlib.Path(d)
        with paths.set_temp_cache(d):
            us = set(u for u, c in islice(valid_urls, FEW))
            urls = {u: download_file(u, cache=True) for u in us}
            files = set(d.iterdir())
            monkeypatch.setattr(os, "mkdir", no_mkdir)
            yield urls
            assert set(d.iterdir()) == files
            check_download_cache(check_hashes=True) 

# 需要导入模块: from astropy.utils import data [as 别名]
# 或者: from astropy.utils.data import download_file [as 别名]
def test_download_file_threaded_many(b, temp_cache, valid_urls):
    """Hammer download_file with multiple threaded requests.

    The goal is to stress-test the locking system. Normal parallel downloading
    also does this but coverage tools lose track of which paths are explored.

    """
    create_cache_with_backend(b)

    urls = list(islice(valid_urls, N_THREAD_HAMMER))
    with ThreadPoolExecutor(max_workers=len(urls)) as P:
        r = list(P.map(lambda u: download_file(u, cache=True),
                       [u for (u, c) in urls]))
    check_download_cache()
    assert len(r) == len(urls)
    for r, (u, c) in zip(r, urls):
        assert get_file_contents(r) == c 

# 需要导入模块: from astropy.utils import data [as 别名]
# 或者: from astropy.utils.data import download_file [as 别名]
def test_download_file_local_cache_survives(tmpdir, temp_cache, use_cache):
    """Confirm that downloading a local file does not delete it.

    When implemented with urlretrieve (rather than urlopen) local files are
    not copied to create temporaries, so importing them to the cache deleted
    the original from wherever it was in the filesystem. I lost some built-in
    astropy data.

    """
    fn = tmpdir / "file"
    contents = "some text"
    with open(fn, "w") as f:
        f.write(contents)
    u = url_to(fn)
    f = download_file(u, cache=use_cache)
    assert fn not in _tempfilestodel, "File should not be deleted!"
    assert os.path.isfile(fn), "File should not be deleted!"
    assert get_file_contents(f) == contents 

# 需要导入模块: from astropy.utils import data [as 别名]
# 或者: from astropy.utils.data import download_file [as 别名]
def test_update_url(tmpdir, temp_cache):
    with TemporaryDirectory(dir=tmpdir) as d:
        f_name = os.path.join(d, "f")
        with open(f_name, "w") as f:
            f.write("old")
        f_url = url_to(f.name)
        assert get_file_contents(download_file(f_url, cache=True)) == "old"
        with open(f_name, "w") as f:
            f.write("new")
        assert get_file_contents(download_file(f_url, cache=True)) == "old"
        assert get_file_contents(download_file(f_url, cache="update")) == "new"
    # Now the URL doesn't exist any more.
    assert not os.path.exists(f_name)
    with pytest.raises(urllib.error.URLError):
        # Direct download should fail
        download_file(f_url, cache=False)
    assert get_file_contents(download_file(f_url, cache=True)) == "new", \
        "Cached version should still exist"
    with pytest.raises(urllib.error.URLError):
        # cannot download new version to check for updates
        download_file(f_url, cache="update")
    assert get_file_contents(download_file(f_url, cache=True)) == "new", \
        "Failed update should not remove the current version" 

# 需要导入模块: from astropy.utils import data [as 别名]
# 或者: from astropy.utils.data import download_file [as 别名]
def test_download_cache():

    download_dir = _get_download_cache_locs()[0]

    # Download the test URL and make sure it exists, then clear just that
    # URL and make sure it got deleted.
    fnout = download_file(TESTURL, cache=True)
    assert os.path.isdir(download_dir)
    assert os.path.isfile(fnout)
    clear_download_cache(TESTURL)
    assert not os.path.exists(fnout)

    # Clearing download cache succeeds even if the URL does not exist.
    clear_download_cache("http://this_was_never_downloaded_before.com")

    # Make sure lockdir was released
    lockdir = os.path.join(download_dir, "lock")
    assert not os.path.isdir(lockdir), "Cache dir lock was not released!" 

# 需要导入模块: from astropy.utils import data [as 别名]
# 或者: from astropy.utils.data import download_file [as 别名]
def test_check_download_cache(tmpdir, temp_cache, valid_urls, invalid_urls):
    testurl, testurl_contents = next(valid_urls)
    testurl2, testurl2_contents = next(valid_urls)

    zip_file_name = os.path.join(tmpdir, "the.zip")
    clear_download_cache()
    check_download_cache()

    download_file(testurl, cache=True)
    # normal files probably corresponding to the urlmap
    normal = check_download_cache()
    download_file(testurl2, cache=True)
    assert check_download_cache() == normal

    export_download_cache(zip_file_name, [testurl, testurl2])
    assert check_download_cache(check_hashes=True) == normal

    clear_download_cache(testurl2)
    assert check_download_cache() == normal

    import_download_cache(zip_file_name, [testurl])
    assert check_download_cache(check_hashes=True) == normal 

# 需要导入模块: from astropy.utils import data [as 别名]
# 或者: from astropy.utils.data import download_file [as 别名]
def test_export_import_roundtrip_one(tmpdir, temp_cache, valid_urls):
    testurl, contents = next(valid_urls)
    f = download_file(testurl, cache=True, show_progress=False)
    assert get_file_contents(f) == contents
    normal = check_download_cache()
    initial_urls_in_cache = set(get_cached_urls())
    zip_file_name = os.path.join(tmpdir, "the.zip")

    export_download_cache(zip_file_name, [testurl])
    clear_download_cache(testurl)
    import_download_cache(zip_file_name)
    assert is_url_in_cache(testurl)
    assert set(get_cached_urls()) == initial_urls_in_cache
    assert (
        get_file_contents(download_file(testurl, cache=True, show_progress=False))
        == contents
    )
    assert check_download_cache(check_hashes=True) == normal 

# 需要导入模块: from astropy.utils import data [as 别名]
# 或者: from astropy.utils.data import download_file [as 别名]
def test_export_import_roundtrip_different_location(tmpdir, valid_urls):
    original_cache = tmpdir / "original"
    os.mkdir(original_cache)
    zip_file_name = tmpdir / "the.zip"

    urls = list(islice(valid_urls, FEW))
    initial_urls_in_cache = set(u for (u, c) in urls)
    with paths.set_temp_cache(original_cache):
        for u, c in urls:
            download_file(u, cache=True)
        assert set(get_cached_urls()) == initial_urls_in_cache
        export_download_cache(zip_file_name)

    new_cache = tmpdir / "new"
    os.mkdir(new_cache)
    with paths.set_temp_cache(new_cache):
        import_download_cache(zip_file_name)
        check_download_cache(check_hashes=True)
        assert set(get_cached_urls()) == initial_urls_in_cache
        for (u, c) in urls:
            assert get_file_contents(download_file(u, cache=True)) == c 

# 需要导入模块: from astropy.utils import data [as 别名]
# 或者: from astropy.utils.data import download_file [as 别名]
def test_cache_not_relocatable(tmpdir, valid_urls):
    u, c = next(valid_urls)
    d1 = tmpdir / "1"
    d2 = tmpdir / "2"
    os.mkdir(d1)
    with paths.set_temp_cache(d1):
        p1 = download_file(u, cache=True)
        assert is_url_in_cache(u)
        assert get_file_contents(p1) == c
        shutil.copytree(d1, d2)
        clear_download_cache()
    # this will not work! The filenames listed in the shelve are absolute
    # and so point back to the first cache
    with paths.set_temp_cache(d2):
        assert is_url_in_cache(u)
        p2 = download_file(u, cache=True)
        assert p1 == p2
        assert not os.path.exists(p2)
        with pytest.raises(RuntimeError):
            clear_download_cache(p2)
        with pytest.raises(CacheDamaged):
            check_download_cache() 

# 需要导入模块: from astropy.utils import data [as 别名]
# 或者: from astropy.utils.data import download_file [as 别名]
def test_wrong_backend_reports_useful_error(b1b2, temp_cache, valid_urls):
    b1, b2 = b1b2
    create_cache_with_backend(b1)
    for u, c in islice(valid_urls, FEW):
        download_file(u, cache=True)
    with shelve_backend(b2):
        for u, c in islice(valid_urls, FEW):
            with pytest.warns(WrongDBMModuleWarning) as w:
                f = download_file(u, cache=True)
                assert get_file_contents(f) == c
            for wi in w:
                assert "module" in str(wi.message.args[0])
                assert b1 in str(wi.message.args[0])
        with pytest.raises(CacheDamaged):
            with pytest.warns(WrongDBMModuleWarning):
                check_download_cache()
        # We should still be able to wipe out the cache!
        clear_download_cache()


# What happens when the lock can't be obtained in a timely manner? 

# 需要导入模块: from astropy.utils import data [as 别名]
# 或者: from astropy.utils.data import download_file [as 别名]
def test_lock_unavailable_raises_runtimeerror_and_clear_frees_lock(
        temp_cache, valid_urls):
    u, c = next(valid_urls)
    download_file(u, cache=True)
    with _cache_lock("astropy", need_write=True):
        with conf.set_temp("download_cache_lock_attempts", 0):
            with pytest.raises(RuntimeError):
                with _cache_lock("astropy", need_write=True):
                    pass
            # Ensure that the cache doesn't accidentally get unlocked!
            with pytest.raises(RuntimeError):
                with _cache_lock("astropy", need_write=True):
                    pass
            # Trying to do anything should raise an exception
            with pytest.raises(RuntimeError):
                is_url_in_cache(u)
            with pytest.raises(RuntimeError):
                download_file(u, cache=True, sources=[])
            with pytest.raises(RuntimeError):
                download_file(u, cache=True)
            clear_download_cache()  # breaks lock
            is_url_in_cache(u)  # False but doesn't raise an exception
    # Exiting the lock should succeed even though it was broken open 

# 需要导入模块: from astropy.utils import data [as 别名]
# 或者: from astropy.utils.data import download_file [as 别名]
def _get_kernel(value):
    """
    Try importing jplephem, download/retrieve from cache the Satellite Planet
    Kernel corresponding to the given ephemeris.
    """
    if value is None or value.lower() == 'builtin':
        return None

    try:
        from jplephem.spk import SPK
    except ImportError:
        raise ImportError("Solar system JPL ephemeris calculations require "
                          "the jplephem package "
                          "(https://pypi.org/project/jplephem/)")

    if value.lower() == 'jpl':
        value = DEFAULT_JPL_EPHEMERIS

    if value.lower() in ('de430', 'de432s'):
        value = ('https://naif.jpl.nasa.gov/pub/naif/generic_kernels'
                 '/spk/planets/{:s}.bsp'.format(value.lower()))

    elif os.path.isfile(value):
        return SPK.open(value)

    else:
        try:
            urlparse(value)
        except Exception:
            raise ValueError('{} was not one of the standard strings and '
                             'could not be parsed as a file path or URL'.format(value))

    return SPK.open(download_file(value, cache=True)) 

# 需要导入模块: from astropy.utils import data [as 别名]
# 或者: from astropy.utils.data import download_file [as 别名]
def readonly_cache(tmpdir, valid_urls):
    with TemporaryDirectory(dir=tmpdir) as d:
        # other fixtures use the same tmpdir so we need a subdirectory
        # to make into the cache
        d = pathlib.Path(d)
        with paths.set_temp_cache(d):
            us = set(u for u, c in islice(valid_urls, FEW))
            urls = {u: download_file(u, cache=True) for u in us}
            files = set(d.iterdir())
            with readonly_dir(d):
                if not is_dir_readonly(d):
                    pytest.skip("Unable to make directory readonly")
                yield urls
            assert set(d.iterdir()) == files
            check_download_cache(check_hashes=True)