Python shutil.copyfileobj方法代码示例

# 需要导入模块: import shutil [as 别名]
# 或者: from shutil import copyfileobj [as 别名]
def _get_data(self):
        archive_file_name, archive_hash = self._archive_file
        data_file_name, data_hash = self._data_file[self._segment]
        path = os.path.join(self._root, data_file_name)
        if not os.path.exists(path) or not check_sha1(path, data_hash):
            namespace = 'gluon/dataset/'+self._namespace
            downloaded_file_path = download(_get_repo_file_url(namespace, archive_file_name),
                                            path=self._root,
                                            sha1_hash=archive_hash)

            with zipfile.ZipFile(downloaded_file_path, 'r') as zf:
                for member in zf.namelist():
                    filename = os.path.basename(member)
                    if filename:
                        dest = os.path.join(self._root, filename)
                        with zf.open(member) as source, \
                             open(dest, "wb") as target:
                            shutil.copyfileobj(source, target)

        data, label = self._read_batch(path)

        self._data = nd.array(data, dtype=data.dtype).reshape((-1, self._seq_len))
        self._label = nd.array(label, dtype=label.dtype).reshape((-1, self._seq_len)) 

# 需要导入模块: import shutil [as 别名]
# 或者: from shutil import copyfileobj [as 别名]
def sendImageWithUrl(self, to_, url):
        """Send a image with given image url

        :param url: image url to send
        """
        path = '%s/pythonLine-%1.data' % (tempfile.gettempdir(), randint(0, 9))


        r = requests.get(url, stream=True)
        if r.status_code == 200:
            with open(path, 'w') as f:
                shutil.copyfileobj(r.raw, f)
        else:
            raise Exception('Download image failure.')

        try:
            self.sendImage(to_, path)
        except Exception as e:
            raise e 

# 需要导入模块: import shutil [as 别名]
# 或者: from shutil import copyfileobj [as 别名]
def DownloadFile(self, fileurl, dlfile):
    """Downloads a given file to a given path/filename.

    Args:
      fileurl: String with URL of file to download.
      dlfile: String with path of file to be written to.
    Raises:
      OSError: If file cannot be opened/written to, function raises OSError.
      URLError: If URL cannot be opened, fucntion raises URLError.
    """
    if not os.path.isfile(dlfile) or dlfile == TMPINDEX:
      print 'Downloading %s ...' % fileurl
      file_to_dl = urllib2.urlopen(fileurl)
      tmpfile = open(dlfile, 'wb')
      shutil.copyfileobj(file_to_dl, tmpfile)
    else:
      print '%s exists' % dlfile 

# 需要导入模块: import shutil [as 别名]
# 或者: from shutil import copyfileobj [as 别名]
def download(directory, filename):
  """Download (and unzip) a file from the MNIST dataset if not already done."""
  filepath = os.path.join(directory, filename)
  if tf.gfile.Exists(filepath):
    return filepath
  if not tf.gfile.Exists(directory):
    tf.gfile.MakeDirs(directory)
  # CVDF mirror of http://yann.lecun.com/exdb/mnist/
  url = 'https://storage.googleapis.com/cvdf-datasets/mnist/' + filename + '.gz'
  _, zipped_filepath = tempfile.mkstemp(suffix='.gz')
  print('Downloading %s to %s' % (url, zipped_filepath))
  urllib.request.urlretrieve(url, zipped_filepath)
  with gzip.open(zipped_filepath, 'rb') as f_in, \
      tf.gfile.Open(filepath, 'wb') as f_out:
    shutil.copyfileobj(f_in, f_out)
  os.remove(zipped_filepath)
  return filepath 

# 需要导入模块: import shutil [as 别名]
# 或者: from shutil import copyfileobj [as 别名]
def download_image_requests_to_dir(img_url, dir_name):
    dir_utils.clear_directory(f'{dir_utils.get_temp_med_dir()}/{dir_name}')
    img_ext = img_url.rsplit('.', 1)[1]
    s = requests.Session()
    r = s.get(img_url)
    if r.status_code == 200:
        with open(f"{dir_utils.get_temp_med_dir()}/{dir_name}/_image.{img_ext}", 'wb') as f:
            r.raw.decode_content = True
            shutil.copyfileobj(r.raw, f)
        dprint(f"Downloaded image from: {img_url}")
    else:
        dprint(f"{r.status_code} Error! - {img_url}")
    if img_ext == 'png':
        dprint(f"Fixing image to force jpg conversion: {img_url}")
        img_fix = Image.open(f"{dir_utils.get_temp_med_dir()}/{dir_name}/_image.{img_ext}")
        img_fix.convert('RGB').save(f"{dir_utils.get_temp_med_dir()}/{dir_name}/_image.jpg")
        dir_utils.remove_file("_image.png", f'{dir_utils.get_temp_med_dir()}/{dir_name}')
    dprint(f"Downloaded image from: {img_url}") 

# 需要导入模块: import shutil [as 别名]
# 或者: from shutil import copyfileobj [as 别名]
def copyfileobj(src, dst, length=None):
    """Copy length bytes from fileobj src to fileobj dst.
       If length is None, copy the entire content.
    """
    if length == 0:
        return
    if length is None:
        shutil.copyfileobj(src, dst)
        return

    BUFSIZE = 16 * 1024
    blocks, remainder = divmod(length, BUFSIZE)
    for b in range(blocks):
        buf = src.read(BUFSIZE)
        if len(buf) < BUFSIZE:
            raise IOError("end of file reached")
        dst.write(buf)

    if remainder != 0:
        buf = src.read(remainder)
        if len(buf) < remainder:
            raise IOError("end of file reached")
        dst.write(buf)
    return 

# 需要导入模块: import shutil [as 别名]
# 或者: from shutil import copyfileobj [as 别名]
def addfile(self, tarinfo, fileobj=None):
        """Add the TarInfo object `tarinfo' to the archive. If `fileobj' is
           given, tarinfo.size bytes are read from it and added to the archive.
           You can create TarInfo objects using gettarinfo().
           On Windows platforms, `fileobj' should always be opened with mode
           'rb' to avoid irritation about the file size.
        """
        self._check("aw")

        tarinfo = copy.copy(tarinfo)

        buf = tarinfo.tobuf(self.format, self.encoding, self.errors)
        self.fileobj.write(buf)
        self.offset += len(buf)

        # If there's data to follow, append it.
        if fileobj is not None:
            copyfileobj(fileobj, self.fileobj, tarinfo.size)
            blocks, remainder = divmod(tarinfo.size, BLOCKSIZE)
            if remainder > 0:
                self.fileobj.write(NUL * (BLOCKSIZE - remainder))
                blocks += 1
            self.offset += blocks * BLOCKSIZE

        self.members.append(tarinfo) 

# 需要导入模块: import shutil [as 别名]
# 或者: from shutil import copyfileobj [as 别名]
def download_metadata_file(url, outputdir, program):
    """Download and unzip the catalogue files."""
    zipped_index_path = os.path.join(outputdir, 'index_' + program + '.csv.gz')
    if not os.path.isfile(zipped_index_path):
        if not os.path.exists(os.path.dirname(zipped_index_path)):
            os.makedirs(os.path.dirname(zipped_index_path))
        print("Downloading Metadata file...")
        content = urlopen(url)
        with open(zipped_index_path, 'wb') as f:
            shutil.copyfileobj(content, f)
    index_path = os.path.join(outputdir, 'index_' + program + '.csv')
    if not os.path.isfile(index_path):
        print("Unzipping Metadata file...")
        with gzip.open(zipped_index_path) as gzip_index, open(index_path, 'wb') as f:
            shutil.copyfileobj(gzip_index, f)
    return index_path 

# 需要导入模块: import shutil [as 别名]
# 或者: from shutil import copyfileobj [as 别名]
def compress_file_with_gzip(file_name, tmp_dir):
        """Compresses a file with GZIP.

        Args:
            file_name: Local path to file to be compressed.
            tmp_dir: Temporary directory where an GZIP file will be created.

        Returns:
            A tuple of gzip file name and size.
        """
        logger = getLogger(__name__)
        base_name = os.path.basename(file_name)
        gzip_file_name = os.path.join(tmp_dir, base_name + '_c.gz')
        logger.debug('gzip file: %s, original file: %s', gzip_file_name,
                     file_name)
        fr = open(file_name, 'rb')
        fw = gzip.GzipFile(gzip_file_name, 'wb')
        shutil.copyfileobj(fr, fw)
        fw.close()
        fr.close()
        SnowflakeFileUtil.normalize_gzip_header(gzip_file_name)

        statinfo = os.stat(gzip_file_name)
        return gzip_file_name, statinfo.st_size 

# 需要导入模块: import shutil [as 别名]
# 或者: from shutil import copyfileobj [as 别名]
def do_GET(self):
        client_address = self.client_address[0]
        logger.info('Serving transcoded media file to {} ...'.format(
            client_address))

        self.send_head()
        path = self.translate_path(self.path)
        command = VLCEncoderSettings.command(path)
        logger.info('Launching {}'.format(command))

        try:
            with open(os.devnull, 'w') as dev_null:
                encoder_process = subprocess.Popen(
                    command, stdout=subprocess.PIPE, stderr=dev_null)
                shutil.copyfileobj(encoder_process.stdout, self.wfile)
        except:
            logger.info('Connection from {} closed.'.format(client_address))
            logger.debug(traceback.format_exc())
        finally:
            pid = encoder_process.pid
            logger.info('Terminating process {}'.format(pid))
            try:
                os.kill(pid, signal.SIGKILL)
            except:
                pass 

# 需要导入模块: import shutil [as 别名]
# 或者: from shutil import copyfileobj [as 别名]
def sendImageWithURL2(self, to, url):
        """Send a image with given image url

        :param url: image url to send
        """
        path = 'tmp/pythonLine.data'

        r = requests.get(url, stream=True)
        if r.status_code == 200:
            with open(path, 'wb') as f:
                shutil.copyfileobj(r.raw, f)
        else:
            raise Exception('Download image failure.')

        try:
            self.sendImage(to, path)
        except Exception as e:
            raise e 

# 需要导入模块: import shutil [as 别名]
# 或者: from shutil import copyfileobj [as 别名]
def download_tile(name):
        """
        This function will download and extract the tile with the given name.
        The data is stored in the path pointed to by the :code:`_data_path`
        attribute of the module.

        Args:
            name(str): The name of the tile to download.
        """
        base_url = "https://dds.cr.usgs.gov/srtm/version2_1/SRTM30"
        url = base_url + "/" + name + "/" + name + ".dem.zip"
        r = urllib.request.urlopen(url)

        filename = os.path.join(_get_data_path(), name + ".dem.zip")
        path = os.path.join(filename)
        with open(path, 'wb') as f:
            shutil.copyfileobj(r, f)

        # Extract zip file.
        with zipfile.ZipFile(filename, "r") as zip_ref:
            zip_ref.extractall(os.path.dirname(filename)) 

# 需要导入模块: import shutil [as 别名]
# 或者: from shutil import copyfileobj [as 别名]
def write_output(ts, args):
    """
    Adds provenance information to the specified tree sequence (ensuring that the
    output is reproducible) and write the resulting tree sequence to output.
    """
    tables = ts.dump_tables()
    logger.debug("Updating provenance")
    provenance = get_provenance_dict()
    tables.provenances.add_row(json.dumps(provenance))
    ts = tables.tree_sequence()
    if args.output is None:
        # There's no way to get tskit to write directly to stdout, so we write
        # to a tempfile first.
        with tempfile.TemporaryDirectory() as tmpdir:
            tmpfile = pathlib.Path(tmpdir) / "tmp.trees"
            ts.dump(tmpfile)
            with open(tmpfile, "rb") as f:
                shutil.copyfileobj(f, sys.stdout.buffer)
    else:
        logger.debug(f"Writing to {args.output}")
        ts.dump(args.output) 

# 需要导入模块: import shutil [as 别名]
# 或者: from shutil import copyfileobj [as 别名]
def extractFromVolume(container_urn, volume, imageURNs, destFolder):
    printVolumeInfo(container_urn.original_filename, volume)
    resolver = volume.resolver
    for imageUrn in imageURNs:
        imageUrn = utils.SmartUnicode(imageUrn)

        pathName = next(resolver.QuerySubjectPredicate(volume.urn, imageUrn, volume.lexicon.pathName))

        with resolver.AFF4FactoryOpen(imageUrn) as srcStream:
            if destFolder != "-":
                pathName = escaping.arnPathFragment_from_path(pathName.value)
                while pathName.startswith("/"):
                    pathName = pathName[1:]
                destFile = os.path.join(destFolder, pathName)
                if not os.path.exists(os.path.dirname(destFile)):
                    try:
                        os.makedirs(os.path.dirname(destFile))
                    except OSError as exc:  # Guard against race condition
                        if exc.errno != errno.EEXIST:
                            raise
                with open(destFile, "wb") as destStream:
                    shutil.copyfileobj(srcStream, destStream, length=32 * 2014)
                    print("\tExtracted %s to %s" % (pathName, destFile))
            else:
                shutil.copyfileobj(srcStream, sys.stdout) 

# 需要导入模块: import shutil [as 别名]
# 或者: from shutil import copyfileobj [as 别名]
def download_image(url, path):
    if os.path.exists(path):
        return True

    headers = {
        'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.101 Safari/537.36'
    }
    try:

        r = requests.get(url, stream=True, timeout=9, headers=headers)
        if r.status_code == 200:
            with open(path, 'wb') as f:
                r.raw.decode_content = True
                shutil.copyfileobj(r.raw, f)
                return True
        else:
            print(("Could not download image %s, response %d" % (url, r.status_code)))
    except Exception as e:
        if hasattr(e, 'message'):
            print(("Could not download image %s due to %s" % (url, e.message)))
        else:
            print(("Could not download image %s due to %s" % (url, repr(e))))

    return False