Python request.urlretrieve方法代码示例

# 需要导入模块: from six.moves.urllib import request [as 别名]
# 或者: from six.moves.urllib.request import urlretrieve [as 别名]
def maybe_download(url, filename, prefix, num_bytes=None):
    """Takes an URL, a filename, and the expected bytes, download
    the contents and returns the filename.
    num_bytes=None disables the file size check."""
    local_filename = None
    if not os.path.exists(os.path.join(prefix, filename)):
        try:
            print "Downloading file {}...".format(url + filename)
            with tqdm(unit='B', unit_scale=True, miniters=1, desc=filename) as t:
                local_filename, _ = urlretrieve(url + filename, os.path.join(prefix, filename), reporthook=reporthook(t))
        except AttributeError as e:
            print "An error occurred when downloading the file! Please get the dataset using a browser."
            raise e
    # We have a downloaded file
    # Check the stats and make sure they are ok
    file_stats = os.stat(os.path.join(prefix, filename))
    if num_bytes is None or file_stats.st_size == num_bytes:
        print "File {} successfully loaded".format(filename)
    else:
        raise Exception("Unexpected dataset size. Please get the dataset using a browser.")

    return local_filename 

# 需要导入模块: from six.moves.urllib import request [as 别名]
# 或者: from six.moves.urllib.request import urlretrieve [as 别名]
def maybe_download(filename, expected_bytes, force=False):
    """Download a file if not present, and make sure it's the right size."""
    if force or not os.path.exists(filename):
        print('Attempting to download:', filename)
        filename, _ = urlretrieve(url + filename, filename,
                                  reporthook=download_progress_hook)
        print('\nDownload Complete!')
    statinfo = os.stat(filename)

    if statinfo.st_size == expected_bytes:
        print('Found and verified', filename)
    else:
        raise Exception(
                        'Failed to verify ' + filename + \
                        '. Can you get to it with a browser?')
    return filename 

# 需要导入模块: from six.moves.urllib import request [as 别名]
# 或者: from six.moves.urllib.request import urlretrieve [as 别名]
def retrieve_file_from_url(url):
    """
    Retrieve a file from an URL

    Args:
        url: The URL to retrieve the file from.

    Returns:
        The absolute path of the downloaded file.
    """
    try:
        alias_source, _ = urlretrieve(url)
        # Check for HTTPError in Python 2.x
        with open(alias_source, 'r') as f:
            content = f.read()
            if content[:3].isdigit():
                raise CLIError(ALIAS_FILE_URL_ERROR.format(url, content.strip()))
    except Exception as exception:
        if isinstance(exception, CLIError):
            raise

        # Python 3.x
        raise CLIError(ALIAS_FILE_URL_ERROR.format(url, exception))

    return alias_source 

# 需要导入模块: from six.moves.urllib import request [as 别名]
# 或者: from six.moves.urllib.request import urlretrieve [as 别名]
def maybe_download(filename, expected_bytes, force=False):
  """Download a file if not present, and make sure it's the right size."""
  dest_filename = os.path.join(data_root, filename)
  if force or not os.path.exists(dest_filename):
    print('Attempting to download:', filename) 
    filename, _ = urlretrieve(url + filename, dest_filename, reporthook=download_progress_hook)
    print('\nDownload Complete!')
  statinfo = os.stat(dest_filename)
  if statinfo.st_size == expected_bytes:
    print('Found and verified', dest_filename)
  else:
    raise Exception(
      'Failed to verify ' + dest_filename + '. Can you get to it with a browser?')
  return dest_filename


#num_classes = 10 

# 需要导入模块: from six.moves.urllib import request [as 别名]
# 或者: from six.moves.urllib.request import urlretrieve [as 别名]
def download(url, dst_file_path):
    # Download a file, showing progress
    bar_wrap = [None]

    def reporthook(count, block_size, total_size):
        bar = bar_wrap[0]
        if bar is None:
            bar = progressbar.ProgressBar(
                maxval=total_size,
                widgets=[
                    progressbar.Percentage(),
                    ' ',
                    progressbar.Bar(),
                    ' ',
                    progressbar.FileTransferSpeed(),
                    ' | ',
                    progressbar.ETA(),
                ])
            bar.start()
            bar_wrap[0] = bar
        bar.update(min(count * block_size, total_size))

    request.urlretrieve(url, dst_file_path, reporthook=reporthook) 

# 需要导入模块: from six.moves.urllib import request [as 别名]
# 或者: from six.moves.urllib.request import urlretrieve [as 别名]
def load_bert_vocab(vocab_file):
    global BERT_VOCAB
    if BERT_VOCAB is not None:
        return BERT_VOCAB

    if validate_url(vocab_file):
        print(f'Downloading {vocab_file}')
        vocab_file, _ = urlretrieve(vocab_file)

    vocab = collections.OrderedDict()
    index = 0
    with open(vocab_file, "r") as rf:
        for line in rf:
            token = convert_to_unicode(line)
            if not token:
                break
            token = token.strip()
            vocab[token] = index
            index += 1
    BERT_VOCAB = vocab
    return vocab 

# 需要导入模块: from six.moves.urllib import request [as 别名]
# 或者: from six.moves.urllib.request import urlretrieve [as 别名]
def web_downloader(url, path_to_save=None):
    # Use a class to simulate the nonlocal keyword in 2.7
    class Context: pg = None

    def _report_hook(count, block_size, total_size):
        if Context.pg is None:
            length = int((total_size + block_size - 1) / float(block_size)) if total_size != -1 else 1
            Context.pg = create_progress_bar(length)
        Context.pg.update()

    if not path_to_save:
        path_to_save = "/tmp/data.dload-{}".format(os.getpid())
    try:
        path_to_save, _ = urlretrieve(url, path_to_save, reporthook=_report_hook)
        Context.pg.done()
    except Exception as e:  # this is too broad but there are too many exceptions to handle separately
        raise RuntimeError("failed to download data from [url]: {} [to]: {}".format(url, path_to_save))
    return path_to_save 

# 需要导入模块: from six.moves.urllib import request [as 别名]
# 或者: from six.moves.urllib.request import urlretrieve [as 别名]
def download_exchange_symbols(exchange_name):
    """
    Downloads the exchange's symbols.json from the repository.

    Parameters
    ----------
    exchange_name: str
    environ:

    Returns
    -------
    str

    """
    filename = get_exchange_symbols_filename(exchange_name)
    url = SYMBOLS_URL.format(exchange=exchange_name)
    response = request.urlretrieve(url=url, filename=filename)
    return response 

# 需要导入模块: from six.moves.urllib import request [as 别名]
# 或者: from six.moves.urllib.request import urlretrieve [as 别名]
def download_file(file_url, output_file_dir, expected_size, FORCE=False):
    name = file_url.split('/')[-1]
    file_output_path = os.path.join(output_file_dir, name)
    print('Attempting to download ' + file_url)
    print('File output path: ' + file_output_path)
    print('Expected size: ' + str(expected_size))
    if not os.path.isdir(output_file_dir):
        os.makedirs(output_file_dir)

    if os.path.isfile(file_output_path) and os.stat(file_output_path).st_size == expected_size and not FORCE:
        print('File already downloaded completely!')
        return file_output_path
    else:
        print(' ')
        filename, _ = urlretrieve(file_url, file_output_path, download_hook_function)
        print(' ')
        statinfo = os.stat(filename)
        if statinfo.st_size == expected_size:
            print('Found and verified', filename)
        else:
            raise Exception('Could not download ' + filename)
        return filename 

# 需要导入模块: from six.moves.urllib import request [as 别名]
# 或者: from six.moves.urllib.request import urlretrieve [as 别名]
def download_file(file_url, output_file_dir, expected_size, FORCE=False):
    name = file_url.split('/')[-1]
    file_output_path = os.path.join(output_file_dir, name)
    print('Attempting to download ' + file_url)
    print('File output path: ' + file_output_path)
    print('Expected size: ' + str(expected_size))
    if not os.path.isdir(output_file_dir):
        os.makedirs(output_file_dir)

    if os.path.isfile(file_output_path) and os.stat(file_output_path).st_size == expected_size and not FORCE:
        print('File already downloaded completely!')
        return file_output_path
    else:
        print(' ')
        filename, _ = urlretrieve(file_url, file_output_path, download_hook_function)
        print(' ')
        statinfo = os.stat(filename)
        print(statinfo.st_size)
        if statinfo.st_size == expected_size:
            print('Found and verified', filename)
        else:
            raise Exception('Could not download ' + filename)
        return filename 

# 需要导入模块: from six.moves.urllib import request [as 别名]
# 或者: from six.moves.urllib.request import urlretrieve [as 别名]
def maybe_download(url, filename, prefix, num_bytes=None):
    """Takes an URL, a filename, and the expected bytes, download
    the contents and returns the filename.
    num_bytes=None disables the file size check."""
    local_filename = None
    output_path = os.path.join(prefix, filename)
    if not os.path.exists(output_path):
        try:
            print("Downloading file {} to {}...".format(url + filename, output_path))
            with tqdm(unit='B', unit_scale=True, miniters=1, desc=filename) as t:
                local_filename, _ = urlretrieve(url + filename, output_path, reporthook=reporthook(t))
        except AttributeError as e:
            print("An error occurred when downloading the file! Please get the dataset using a browser.")
            raise e
    # We have a downloaded file
    # Check the stats and make sure they are ok
    file_stats = os.stat(os.path.join(prefix, filename))
    if num_bytes is None or file_stats.st_size == num_bytes:
        print("File {} successfully downloaded to {}.".format(filename, output_path))
    else:
        raise Exception("Unexpected dataset size. Please get the dataset using a browser.")

    return local_filename 

# 需要导入模块: from six.moves.urllib import request [as 别名]
# 或者: from six.moves.urllib.request import urlretrieve [as 别名]
def download_data_url(url, download_dir):
	filename = url.split('/')[-1]
	file_path = os.path.join(download_dir, filename)
	
	if not os.path.exists(file_path):
		os.makedirs(download_dir, exist_ok=True)
		
		print('Download %s to %s' % (url, file_path))
		file_path, _ = request.urlretrieve(
			url=url,
			filename=file_path,
			reporthook=report_download_progress)
		
		print('\nExtracting files')
		if file_path.endswith('.zip'):
			zipfile.ZipFile(file=file_path, mode='r').extractall(download_dir)
		elif file_path.endswith(('.tar.gz', '.tgz')):
			tarfile.open(name=file_path, mode='r:gz').extractall(download_dir) 

# 需要导入模块: from six.moves.urllib import request [as 别名]
# 或者: from six.moves.urllib.request import urlretrieve [as 别名]
def urlretrieve(url, filename, reporthook=None, data=None):
        def chunk_read(response, chunk_size=8192, reporthook=None):
            total_size = response.info().get('Content-Length').strip()
            total_size = int(total_size)
            count = 0
            while 1:
                chunk = response.read(chunk_size)
                if not chunk:
                    break
                count += 1
                if reporthook:
                    reporthook(count, chunk_size, total_size)
                yield chunk

        response = urlopen(url, data)
        with open(filename, 'wb') as fd:
            for chunk in chunk_read(response, reporthook=reporthook):
                fd.write(chunk) 

# 需要导入模块: from six.moves.urllib import request [as 别名]
# 或者: from six.moves.urllib.request import urlretrieve [as 别名]
def urlretrieve(url, filename, reporthook=None, data=None):
        def chunk_read(response, chunk_size=8192, reporthook=None):
            total_size = response.info().get('Content-Length').strip()
            total_size = int(total_size)
            count = 0
            while 1:
                chunk = response.read(chunk_size)
                count += 1
                if not chunk:
                    reporthook(count, total_size, total_size)
                    break
                if reporthook:
                    reporthook(count, chunk_size, total_size)
                yield chunk

        response = urlopen(url, data)
        with open(filename, 'wb') as fd:
            for chunk in chunk_read(response, reporthook=reporthook):
                fd.write(chunk) 

# 需要导入模块: from six.moves.urllib import request [as 别名]
# 或者: from six.moves.urllib.request import urlretrieve [as 别名]
def cached_download(url):
    """Downloads a file and caches it.

    This is different from the original
    :func:`~chainer.dataset.cached_download` in that the download
    progress is reported.
    It downloads a file from the URL if there is no corresponding cache.
    If there is already a cache for the given URL, it just returns the
    path to the cache without downloading the same file.
    Args:
        url (string): URL to download from.
    Returns:
        string: Path to the downloaded file.
    """
    cache_root = os.path.join(get_dataset_root(), '_dl_cache')
    try:
        os.makedirs(cache_root)
    except OSError:
        if not os.path.exists(cache_root):
            raise
    lock_path = os.path.join(cache_root, '_dl_lock')
    urlhash = hashlib.md5(url.encode('utf-8')).hexdigest()
    cache_path = os.path.join(cache_root, urlhash)

    with filelock.FileLock(lock_path):
        if os.path.exists(cache_path):
            return cache_path

    temp_root = tempfile.mkdtemp(dir=cache_root)
    try:
        temp_path = os.path.join(temp_root, 'dl')
        print('Downloading ...')
        print('From: {:s}'.format(url))
        print('To: {:s}'.format(cache_path))
        request.urlretrieve(url, temp_path, _reporthook)
        with filelock.FileLock(lock_path):
            shutil.move(temp_path, cache_path)
    finally:
        shutil.rmtree(temp_root)

    return cache_path