Python tarfile.open函数代码示例

def maybe_download_and_extract(data_url, dest_dir='/tmp/imagenet'):
  """
  Download and extract model tar file.  If the pretrained model we're using doesn't already exist,
   downloads it and unpacks it into a directory.
  :param data_url:  url where tar.gz file exists
  :param dest_dir:  destination directory untar to
  :return:
  """
  if not os.path.exists(dest_dir):
    os.makedirs(dest_dir)
  filename = data_url.split('/')[-1]
  filepath = os.path.join(dest_dir, filename)
  if not os.path.exists(filepath):

    def _progress(count, block_size, total_size):
      sys.stdout.write('\r>> Downloading %s %.1f%%' %
                       (filename,
                        float(count * block_size) / float(total_size) * 100.0))
      sys.stdout.flush()

    filepath, _ = urllib.request.urlretrieve(data_url,
                                             filepath,
                                             _progress)
    print()
    statinfo = os.stat(filepath)
    print('Successfully downloaded', filename, statinfo.st_size, 'bytes.')
    tarfile.open(filepath, 'r:gz').extractall(dest_dir)

def maybe_download_and_extract():
  """Download and extract model tar file.

  If the pretrained model we're using doesn't already exist, this function
  downloads it from the TensorFlow.org website and unpacks it into a directory.
  """
  dest_directory = FLAGS.model_dir
  if not os.path.exists(dest_directory):
    os.makedirs(dest_directory)
  filename = DATA_URL.split('/')[-1]
  filepath = os.path.join(dest_directory, filename)
  if not os.path.exists(filepath):

    def _progress(count, block_size, total_size):
      sys.stdout.write('\r>> Downloading %s %.1f%%' %
                       (filename,
                        float(count * block_size) / float(total_size) * 100.0))
      sys.stdout.flush()

    filepath, _ = urllib.request.urlretrieve(DATA_URL,
                                             filepath,
                                             _progress)
    print()
    statinfo = os.stat(filepath)
    print('Successfully downloaded', filename, statinfo.st_size, 'bytes.')
  tarfile.open(filepath, 'r:gz').extractall(dest_directory)

    def setUp(self):
        super(ImportTestCase, self).setUp()
        self.url = reverse_course_url('import_handler', self.course.id)
        self.content_dir = path(tempfile.mkdtemp())

        def touch(name):
            """ Equivalent to shell's 'touch'"""
            with file(name, 'a'):
                os.utime(name, None)

        # Create tar test files -----------------------------------------------
        # OK course:
        good_dir = tempfile.mkdtemp(dir=self.content_dir)
        os.makedirs(os.path.join(good_dir, "course"))
        with open(os.path.join(good_dir, "course.xml"), "w+") as f:
            f.write('<course url_name="2013_Spring" org="EDx" course="0.00x"/>')

        with open(os.path.join(good_dir, "course", "2013_Spring.xml"), "w+") as f:
            f.write('<course></course>')

        self.good_tar = os.path.join(self.content_dir, "good.tar.gz")
        with tarfile.open(self.good_tar, "w:gz") as gtar:
            gtar.add(good_dir)

        # Bad course (no 'course.xml' file):
        bad_dir = tempfile.mkdtemp(dir=self.content_dir)
        touch(os.path.join(bad_dir, "bad.xml"))
        self.bad_tar = os.path.join(self.content_dir, "bad.tar.gz")
        with tarfile.open(self.bad_tar, "w:gz") as btar:
            btar.add(bad_dir)

        self.unsafe_common_dir = path(tempfile.mkdtemp(dir=self.content_dir))

def install_from_source(setuptools_source, pip_source):
    setuptools_temp_dir = tempfile.mkdtemp('-setuptools', 'ptvs-')
    pip_temp_dir = tempfile.mkdtemp('-pip', 'ptvs-')
    cwd = os.getcwd()

    try:
        os.chdir(setuptools_temp_dir)
        print('Downloading setuptools from ' + setuptools_source)
        sys.stdout.flush()
        setuptools_package, _ = urlretrieve(setuptools_source, 'setuptools.tar.gz')

        package = tarfile.open(setuptools_package)
        try:
            safe_members = [m for m in package.getmembers() if not m.name.startswith(('..', '\\'))]
            package.extractall(setuptools_temp_dir, members=safe_members)
        finally:
            package.close()

        extracted_dirs = [d for d in os.listdir(setuptools_temp_dir) if os.path.exists(os.path.join(d, 'setup.py'))]
        if not extracted_dirs:
            raise OSError("Failed to find setuptools's setup.py")
        extracted_dir = extracted_dirs[0]

        print('\nInstalling from ' + extracted_dir)
        sys.stdout.flush()
        os.chdir(extracted_dir)
        subprocess.check_call(
            EXECUTABLE + ['setup.py', 'install', '--single-version-externally-managed', '--record', 'setuptools.txt']
        )

        os.chdir(pip_temp_dir)
        print('Downloading pip from ' + pip_source)
        sys.stdout.flush()
        pip_package, _ = urlretrieve(pip_source, 'pip.tar.gz')

        package = tarfile.open(pip_package)
        try:
            safe_members = [m for m in package.getmembers() if not m.name.startswith(('..', '\\'))]
            package.extractall(pip_temp_dir, members=safe_members)
        finally:
            package.close()

        extracted_dirs = [d for d in os.listdir(pip_temp_dir) if os.path.exists(os.path.join(d, 'setup.py'))]
        if not extracted_dirs:
            raise OSError("Failed to find pip's setup.py")
        extracted_dir = extracted_dirs[0]

        print('\nInstalling from ' + extracted_dir)
        sys.stdout.flush()
        os.chdir(extracted_dir)
        subprocess.check_call(
            EXECUTABLE + ['setup.py', 'install', '--single-version-externally-managed', '--record', 'pip.txt']
        )

        print('\nInstallation Complete')
        sys.stdout.flush()
    finally:
        os.chdir(cwd)
        shutil.rmtree(setuptools_temp_dir, ignore_errors=True)
        shutil.rmtree(pip_temp_dir, ignore_errors=True)

def unpack_maflib(directory):
    with _Cleaner(directory) as c:
        content = _read_archive(__file__)

        os.makedirs(os.path.join(directory, 'maflib'))
        os.chdir(directory)

        bz2_name = TAR_NAME + '.bz2'
        with open(bz2_name, 'wb') as f:
            f.write(content)

        try:
            t = tarfile.open(bz2_name)
        except:
            try:
                os.system('bunzip2 ' + bz2_name)
                t = tarfile.open(TAR_NAME)
            except:
                raise Exception('Cannot extract maflib. Check that python bz2 module or bunzip2 command is available.')

        try:
            t.extractall()
        finally:
            t.close()

        try:
            os.remove(bz2_name)
            os.remove(TAR_NAME)
        except:
            pass

        maflib_path = os.path.abspath(os.getcwd())
        return maflib_path

    def __call__(self, path, target):
        """Extract C{path} into C{target} using the C{zipfile} module.

        @note: No need to use C{tarfile.is_tarfile} because we want an
        exception on failure anyway."""
        import tarfile
        tarfile.open(path, 'r').extractall(target)

def unpack(filename, destination):
    dirname = ''
    print('Extracting {0}'.format(filename))
    if filename.endswith('tar.gz'):
        tfile = tarfile.open(filename, 'r:gz')
        tfile.extractall(destination)
        dirname = tfile.getnames()[0]
    elif filename.endswith('tar.bz2'):
        tfile = tarfile.open(filename, 'r:bz2')
        tfile.extractall(destination)
        dirname = tfile.getnames()[0]
    elif filename.endswith('zip'):
        zfile = zipfile.ZipFile(filename)
        zfile.extractall(destination)
        dirname = zfile.namelist()[0]
    else:
        raise NotImplementedError('Unsupported archive type')

    # a little trick to rename tool directories so they don't contain version number
    rename_candidate = re.match(r'^([a-z][^\-]*\-*)+', dirname)

    if rename_candidate is not None:
        rename_to = rename_candidate.group(0).encode('ascii').strip('-')
        if rename_to != dirname:
            print('Renaming {0} to {1}'.format(dirname, rename_to))
            if os.path.isdir(rename_to):
                shutil.rmtree(rename_to)
            shutil.move(dirname, rename_to)

    def extract(self):
        # initialize the progress bar
        self.progressbar.set_fraction(0)
        self.progressbar.set_text(_('Installing'))
        self.progressbar.show()
        self.refresh_gtk()

        extracted = False
        try:
            if self.common.paths['tarball_file'][-2:] == 'xz':
                # if tarball is .tar.xz
                xz = lzma.LZMAFile(self.common.paths['tarball_file'])
                tf = tarfile.open(fileobj=xz)
                tf.extractall(self.common.paths['tbb']['dir'])
                extracted = True
            else:
                # if tarball is .tar.gz
                if tarfile.is_tarfile(self.common.paths['tarball_file']):
                    tf = tarfile.open(self.common.paths['tarball_file'])
                    tf.extractall(self.common.paths['tbb']['dir'])
                    extracted = True
        except:
            pass

        if not extracted:
            self.set_gui('task', _("Tor Browser Launcher doesn't understand the file format of {0}".format(self.common.paths['tarball_file'])), ['start_over'], False)
            self.clear_ui()
            self.build_ui()
            return

        # installation is finished, so save installed_version
        self.common.settings['installed_version'] = self.common.settings['latest_version']
        self.common.save_settings()

        self.run_task()

def download_20newsgroups(target_dir, cache_path):
    """ Download the 20Newsgroups data and convert is in a zipped pickle
        storage.
    """
    archive_path = os.path.join(target_dir, ARCHIVE_NAME)
    train_path = os.path.join(target_dir, TRAIN_FOLDER)
    test_path = os.path.join(target_dir, TEST_FOLDER)

    if not os.path.exists(target_dir):
        os.makedirs(target_dir)

    if not os.path.exists(archive_path):
        logger.warn("Downloading dataset from %s (14 MB)", URL)
        opener = urllib.urlopen(URL)
        open(archive_path, 'wb').write(opener.read())

    logger.info("Decompressing %s", archive_path)
    tarfile.open(archive_path, "r:gz").extractall(path=target_dir)
    os.remove(archive_path)

    # Store a zipped pickle
    cache = dict(
            train=load_files(train_path),
            test=load_files(test_path)
        )
    open(cache_path, 'wb').write(pickle.dumps(cache).encode('zip'))
    shutil.rmtree(target_dir)
    return cache

 def datafiles(self):
     """ Get list of readable datafiles from asset (multiple filenames if tar or hdf file) """
     path = os.path.dirname(self.filename)
     indexfile = os.path.join(path, self.filename + '.index')
     if os.path.exists(indexfile):
         datafiles = File2List(indexfile)
         if len(datafiles) > 0:
             return datafiles
     try:
         if tarfile.is_tarfile(self.filename):
             tfile = tarfile.open(self.filename)
             tfile = tarfile.open(self.filename)
             datafiles = tfile.getnames()
         elif zipfile.is_zipfile(self.filename):
             zfile = zipfile.ZipFile(self.filename)
             datafiles = ['/vsizip/' + os.path.join(self.filename, f)
                          for f in zfile.namelist()]
         else:
             # Try subdatasets
             fh = gdal.Open(self.filename)
             sds = fh.GetSubDatasets()
             datafiles = [s[0] for s in sds]
         if len(datafiles) > 0:
             List2File(datafiles, indexfile)
             return datafiles
         else:
             return [self.filename]
     except Exception as e:
         raise Exception('Problem accessing asset(s) in {}\n ({})'
                         .format(self.filename, e))

def _downloadAndExtractTarball(tarbalUrl, targetDir):
    try:
        remoteFile = urllib2.urlopen(tarbalUrl)
    except Exception as ex:
        print 'Failed contacting:', tarbalUrl, ' with error:"', ex, '" retrying...'
        remoteFile = urllib2.urlopen(tarbalUrl)

    try:
        shutil.rmtree(targetDir, ignore_errors=True)
        os.makedirs(targetDir)
    except OSError:
        pass

    localTarBall = os.path.join(targetDir, os.path.basename(tarbalUrl))
    targetFile = open(localTarBall, 'wb')
    while True:
        data = remoteFile.read()
        if not data:
            break
        targetFile.write(data)
    remoteFile.close()
    targetFile.close()

    print 'Expanding tarball:', localTarBall
    tarfile.open(localTarBall, 'r:gz').extractall(targetDir)

def prepareTar(doc, visibleTar=False):
    # Finish the spec
    specOutput = tempfile.NamedTemporaryFile(delete=False)
    doc.finish(outputFilename=specOutput.name)
    # Build the TAR file
    if visibleTar:
        tar = tarfile.open(name="test.tar", mode='w')
    else:
        f = tempfile.NamedTemporaryFile(delete=False)
        tar = tarfile.open(fileobj=f, mode='w')
    tar.add(specOutput.name, arcname="Overview.html")
    additionalFiles = extensions.BSPublishAdditionalFiles(["images", "diagrams", "examples"])
    for fname in additionalFiles:
        try:
            if isinstance(fname, basestring):
                tar.add(fname)
            elif isinstance(fname, list):
                tar.add(fname[0], arcname=fname[1])
        except OSError:
            pass
    tar.close()
    specOutput.close()
    os.remove(specOutput.name)
    if visibleTar:
        return open("test.tar", "rb")
    else:
        f.seek(0)
        return f

def extract_tar_archive(archive_path, destination_directory):
    """
    Extracts the given tarball to the given destination directory.
    It automatically handles the following compressed archives on both Python2
    and Python3.

    - gz
    - xz
    - bz2
    - lzma

    :param archive_path: The path to the archive which should be extracted.
    :type archive_path: string

    :param destination_directory: The directory where the files should be
        extracted to. The directory does not have to exist prior to calling
        this function; it will be automatically created, if not.
    :type destination_directory: string
    """
    # lzma (.lzma and .xz) compressed archives are not automatically
    # uncompressed on python2.
    if archive_path.endswith('.xz') or archive_path.endswith('.lzma'):
        if not six.PY3:
            with contextlib.closing(lzma.LZMAFile(archive_path)) as lzma_file:
                with tarfile.open(fileobj=lzma_file) as archive_file:
                    archive_file.extractall(destination_directory)
            return
    # In all other cases, tarfile handles compression automatically
    with tarfile.open(archive_path) as archive_file:
        archive_file.extractall(destination_directory)

    def test_make_distribution_owner_group(self):
        dist, cmd = self.get_cmd()
        cmd.formats = ['gztar']
        cmd.owner = pwd.getpwuid(0)[0]
        cmd.group = grp.getgrgid(0)[0]
        cmd.ensure_finalized()
        cmd.run()
        archive_name = join(self.tmp_dir, 'dist', 'fake-1.0.tar.gz')
        archive = tarfile.open(archive_name)
        try:
            for member in archive.getmembers():
                self.assertEqual(member.uid, 0)
                self.assertEqual(member.gid, 0)

        finally:
            archive.close()

        dist, cmd = self.get_cmd()
        cmd.formats = ['gztar']
        cmd.ensure_finalized()
        cmd.run()
        archive_name = join(self.tmp_dir, 'dist', 'fake-1.0.tar.gz')
        archive = tarfile.open(archive_name)
        try:
            for member in archive.getmembers():
                self.assertEqual(member.uid, os.getuid())

        finally:
            archive.close()

 def read(self, all_tags=False, self_provides=True, *extra_tags):
     arfile = ar.Ar(fh = self.__file)
     arfile.read()
     debbin = arfile.get_file('debian-binary')
     if debbin is None:
         raise DebError(self.__path, 'no debian binary')
     if debbin.read() != '2.0\n':
         raise DebError(self.__path, 'invalid debian binary format')
     control = arfile.get_file('control.tar.gz')
     if control is not None:
         # XXX: python2.4 relies on a name
         tar = tarfile.open(name='control.tar.gz', fileobj=control)
     else:
         control = arfile.get_file('control.tar.xz')
         if control is None:
             raise DebError(self.__path, 'missing control.tar')
         if not HAVE_LZMA:
             raise DebError(self.__path, 'can\'t open control.tar.xz without python-lzma')
         decompressed = lzma.decompress(control.read())
         tar = tarfile.open(name="control.tar.xz",
                            fileobj=StringIO.StringIO(decompressed))
     try:
         name = './control'
         # workaround for python2.4's tarfile module
         if 'control' in tar.getnames():
             name = 'control'
         control = tar.extractfile(name)
     except KeyError:
         raise DebError(self.__path,
                        'missing \'control\' file in control.tar')
     self.__parse_control(control, all_tags, self_provides, *extra_tags)
     return self