Python shutil.unpack_archive函数代码示例

 def extract_package(self, package):
     if sys.version_info < (3, 5):
         try:
             import lzma
             del lzma
             try:
                 shutil.register_unpack_format('xztar', ['.tar.xz', '.txz'], shutil._unpack_tarfile, [], "xz'ed tar-file")
             except shutil.RegistryError:
                 pass
         except ImportError:
             pass
     target_dir = os.path.join(self.subdir_root, package.get('directory'))
     if os.path.isdir(target_dir):
         return
     extract_dir = self.subdir_root
     # Some upstreams ship packages that do not have a leading directory.
     # Create one for them.
     try:
         package.get('lead_directory_missing')
         os.mkdir(target_dir)
         extract_dir = target_dir
     except KeyError:
         pass
     shutil.unpack_archive(os.path.join(self.cachedir, package.get('source_filename')), extract_dir)
     if package.has_patch():
         shutil.unpack_archive(os.path.join(self.cachedir, package.get('patch_filename')), self.subdir_root)

def read_gtfs(path):
    """
    Given a path (string or pathlib object) to a (zipped) GTFS feed,
    read the feed and return its corresponding Feed instance.

    NOTES:
        - Ignore files that are not valid GTFS; see https://developers.google.com/transit/gtfs/reference/.
        - Ensure that all ID fields that could be string ('stop_id', 'route_id', etc.) are parsed as strings and not as numbers.    
    """
    path = Path(path)
    
    # Unzip feed into temporary directory
    tmp_dir = tempfile.TemporaryDirectory()
    shutil.unpack_archive(str(path), tmp_dir.name, 'zip')

    # Read valid GTFS files into Pandas data frames
    feed_dict = {}
    dtype = {field: str for field in Feed.str_fields} # ensure some string types
    for p in Path(tmp_dir.name).iterdir():        
        name = p.stem
        if name in Feed.gtfs_tables:
            feed_dict[name] = pd.read_csv(p, dtype=dtype)
        
    # Delete temporary directory
    tmp_dir.cleanup()
    
    return Feed(**feed_dict)

 def get_pos_latin_tar(self):
     """Fetch Latin part-of-speech files"""
     orig_files_dir_pos_latin = os.path.join(self.orig_files_dir,
                                             'pos_latin')
     pg_url = 'https://raw.githubusercontent.com/cltk/pos_latin/' \
              'master/pos_latin.tar.gz'
     session = requests.Session()
     session.mount(pg_url, SSLAdapter(ssl.PROTOCOL_TLSv1))
     pg_tar = session.get(pg_url, stream=True)
     pos_latin_file_name = urlsplit(pg_url).path.split('/')[-1]
     pos_latin_file_path = os.path.join(orig_files_dir_pos_latin,
                                        pos_latin_file_name)
     try:
         with open(pos_latin_file_path, 'wb') as new_file:
             new_file.write(pg_tar.content)
             logging.info('Finished writing %s.', pos_latin_file_name)
     except IOError:
         logging.error('Failed to write file %s', pos_latin_file_name)
     compiled_files_dir_pos_latin = os.path.join(self.compiled_files_dir,
                                             'pos_latin')
     if os.path.isdir(compiled_files_dir_pos_latin) is True:
         pass
     else:
         os.mkdir(compiled_files_dir_pos_latin)
         logging.info('Made new directory "pos_latin" at "%s"',
                      compiled_files_dir_pos_latin)
     try:
         shutil.unpack_archive(pos_latin_file_path, compiled_files_dir_pos_latin)
         logging.info('Finished unpacking %s', pos_latin_file_name)
     except IOError:
         logging.info('Failed to unpack %s.', pos_latin_file_name)

 def get_sentence_tokens_greek_tar(self):
     """Fetch algorithm for Greek sentence tokenization"""
     orig_files_dir_tokens_greek = \
         os.path.join(self.orig_files_dir, 'sentence_tokens_greek')
     # make compiled files dir for tokens_greek
     compiled_files_dir_tokens_greek = \
         os.path.join(self.compiled_files_dir, 'sentence_tokens_greek')
     if os.path.isdir(compiled_files_dir_tokens_greek) is True:
         pass
     else:
         os.mkdir(compiled_files_dir_tokens_greek)
     pg_url = 'https://raw.githubusercontent.com/cltk/' \
              'cltk_greek_sentence_tokenizer/master/greek.tar.gz'
     session = requests.Session()
     session.mount(pg_url, SSLAdapter(ssl.PROTOCOL_TLSv1))
     pg_tar = session.get(pg_url, stream=True)
     tokens_greek_file_name = urlsplit(pg_url).path.split('/')[-1]
     tokens_greek_file_path = os.path.join(orig_files_dir_tokens_greek,
                                           tokens_greek_file_name)
     try:
         with open(tokens_greek_file_path, 'wb') as new_file:
             new_file.write(pg_tar.content)
             logging.info('Finished writing %s.', tokens_greek_file_name)
             try:
                 shutil.unpack_archive(tokens_greek_file_path,
                                       compiled_files_dir_tokens_greek)
                 logging.info('Finished unpacking %s.',
                              tokens_greek_file_name)
             except IOError:
                 logging.info('Failed to unpack %s.',
                              tokens_greek_file_name)
     except IOError:
         logging.error('Failed to write file %s', tokens_greek_file_name)

 def get_lacus_curtius_latin_tar(self):
     """Fetch lacus_curtius_latin_tar"""
     orig_files_dir_lacus_curtius_latin = \
         os.path.join(self.orig_files_dir, 'lacus_curtius_latin')
     lc_url = 'https://raw.githubusercontent.com/cltk/' \
              'latin_corpus_lacus_curtius/master/lacus_curtius.tar.gz'
     session = requests.Session()
     session.mount(lc_url, SSLAdapter(ssl.PROTOCOL_TLSv1))
     ll_tar = session.get(lc_url, stream=True)
     lacus_curtius_latin_file_name = urlsplit(lc_url).path.split('/')[-1]
     lacus_curtius_latin_file_path = \
         os.path.join(orig_files_dir_lacus_curtius_latin,
                      lacus_curtius_latin_file_name)
     try:
         with open(lacus_curtius_latin_file_path, 'wb') as new_file:
             new_file.write(ll_tar.content)
             logging.info('Finished writing %s.',
                          lacus_curtius_latin_file_name)
     except IOError:
         logging.error('Failed to write file %s',
                       lacus_curtius_latin_file_name)
     try:
         shutil.unpack_archive(lacus_curtius_latin_file_path,
                               self.compiled_files_dir)
         logging.info('Finished unpacking %s',
                      lacus_curtius_latin_file_name)
     except IOError:
         logging.info('Failed to unpack %s.', lacus_curtius_latin_file_name)

 def get_treebank_perseus_latin_tar(self):
     """Fetch Perseus's Latin treebank files"""
     compiled_files_dir_treebank_perseus_latin = os.path.join(self.compiled_files_dir, 'treebank_perseus_latin')
     if os.path.isdir(compiled_files_dir_treebank_perseus_latin) is True:
         pass
     else:
         os.mkdir(compiled_files_dir_treebank_perseus_latin)
         logging.info('Made new directory at "%s"', compiled_files_dir_treebank_perseus_latin)
     orig_files_dir_treebank_perseus_latin = \
         os.path.join(self.orig_files_dir, 'treebank_perseus_latin')
     pg_url = 'https://raw.githubusercontent.com/cltk/latin_treebank_perseus/master/latin_treebank_perseus.tar.gz'
     session = requests.Session()
     session.mount(pg_url, SSLAdapter(ssl.PROTOCOL_TLSv1))
     pg_tar = session.get(pg_url, stream=True)
     treebank_perseus_latin_file_name = urlsplit(pg_url).path.split('/')[-1]
     treebank_perseus_latin_file_path = \
         os.path.join(orig_files_dir_treebank_perseus_latin,
                      treebank_perseus_latin_file_name)
     try:
         with open(treebank_perseus_latin_file_path, 'wb') as new_file:
             new_file.write(pg_tar.content)
             logging.info('Finished writing %s.',
                          treebank_perseus_latin_file_name)
     except IOError:
         logging.error('Failed to write file %s',
                       treebank_perseus_latin_file_name)
     try:
         shutil.unpack_archive(treebank_perseus_latin_file_path,
                               compiled_files_dir_treebank_perseus_latin)
         logging.info('Finished unpacking %s',
                      treebank_perseus_latin_file_name)
     except IOError:
         logging.info('Failed to unpack %s.',
                      treebank_perseus_latin_file_name)

def unpack(sproj, branch, outdir):
    subprocess.check_call(['git', 'clone', '-b', branch, 'https://github.com/mesonbuild/%s.git' % sproj, outdir])
    usfile = os.path.join(outdir, 'upstream.wrap')
    assert(os.path.isfile(usfile))
    config = configparser.ConfigParser()
    config.read(usfile)
    us_url = config['wrap-file']['source_url']
    us = urllib.request.urlopen(us_url).read()
    h = hashlib.sha256()
    h.update(us)
    dig = h.hexdigest()
    should = config['wrap-file']['source_hash']
    if dig != should:
        print('Incorrect hash on download.')
        print(' expected:', dig)
        print(' obtained:', should)
        return 1
    spdir = os.path.split(outdir)[0]
    ofilename = os.path.join(spdir, config['wrap-file']['source_filename'])
    with open(ofilename, 'wb') as ofile:
        ofile.write(us)
    if 'lead_directory_missing' in config['wrap-file']:
        os.mkdir(outdir)
        shutil.unpack_archive(ofilename, outdir)
    else:
        shutil.unpack_archive(ofilename, spdir)
        extdir = os.path.join(spdir, config['wrap-file']['directory'])
        assert(os.path.isdir(extdir))
        shutil.move(os.path.join(outdir, '.git'), extdir)
        subprocess.check_call(['git', 'reset', '--hard'], cwd=extdir)
        shutil.rmtree(outdir)
        shutil.move(extdir, outdir)
    shutil.rmtree(os.path.join(outdir, '.git'))
    os.unlink(ofilename)

 def get_treebank_perseus_latin_tar(self):
     """Fetch Persus's Latin treebank files"""
     orig_files_dir_treebank_perseus_latin = \
         os.path.join(self.orig_files_dir, 'treebank_perseus_latin')
     pg_url = 'https://raw.githubusercontent.com/kylepjohnson/' \
              'treebank_perseus_latin/master/treebank_perseus_latin.tar.gz'
     session = requests.Session()
     session.mount(pg_url, SSLAdapter(ssl.PROTOCOL_TLSv1))
     pg_tar = session.get(pg_url, stream=True)
     treebank_perseus_latin_file_name = urlsplit(pg_url).path.split('/')[-1]
     treebank_perseus_latin_file_path = \
         os.path.join(orig_files_dir_treebank_perseus_latin,
                      treebank_perseus_latin_file_name)
     try:
         with open(treebank_perseus_latin_file_path, 'wb') as new_file:
             new_file.write(pg_tar.content)
             logging.info('Finished writing %s.',
                          treebank_perseus_latin_file_name)
     except IOError:
         logging.error('Failed to write file %s',
                       treebank_perseus_latin_file_name)
     try:
         shutil.unpack_archive(treebank_perseus_latin_file_path,
                               self.compiled_files_dir)
         logging.info('Finished unpacking %s',
                      treebank_perseus_latin_file_name)
     except IOError:
         logging.info('Failed to unpack %s.',
                      treebank_perseus_latin_file_name)

 def download(self):
     furlo = FBURLopener({})
     try:
         tmpfile, msg = furlo.retrieve(self.url, reporthook=self.rhook)
         print()
     except HTTPError as ex:
         urlcleanup()
         sys.exit(ex)
     except URLError as ex:
         urlcleanup()
         sys.exit(ex)
     if os.path.exists(self.dlpath) and filecmp.cmp(self.dlpath, tmpfile):
         print('You already have the newest version of ' + self.plugin)
         done = True
     else:
         shutil.copyfile(tmpfile, self.dlpath)
         print(self.plugin + ' downloaded.')
         done = False
     urlcleanup()
     if done or self.format == 'jar':
         return
     try:
         shutil.unpack_archive(self.dlpath, self.dest_dir, self.format)
     except ValueError as ex:
         sys.exit('Error: ' + str(ex))

    def restore(self, backup_file, message=str(), delay=0):
        """
        Restores the backup of the world from the given *backup_file*. If
        the backup archive contains the configuration and the server, they
        are restored too.

        Raises: Exception
                A lot of things can go wrong here, so catch *Exception* if you
                want to be sure, you catch everything.
        """
        # Extract the backup in a temporary directory and copy then all things
        # into the EMSM directories.
        with tempfile.TemporaryDirectory() as temp_dir:
            shutil.unpack_archive(
                filename = backup_file,
                extract_dir = temp_dir
                )

            # Stop the world.
            was_online = self.world.is_online()
            if was_online:
                self.world.send_command("say {}".format(message))
                time.sleep(delay)
                self.world.kill_processes()

            # Restore the world.
            manifest = self._get_manifest(temp_dir)

            self._restore_world(manifest, temp_dir)
            self._restore_world_configuration(manifest, temp_dir)
            self._restore_server(manifest, temp_dir)

        if was_online:
            self.world.start()
        return None

def get_file_content( params, target_directory ):
    directory_content = params.get( 'directory_content', [] )
    for content in directory_content:
        target_path = os.path.join( target_directory, content.get( 'subdir', '' ) )
        try:
            os.makedirs( target_path )
        except OSError:
            pass
        if content.get( 'file_source', {}).get( 'file_source_selector', None ) == 'URL':
            ( filename, headers ) = urlretrieve( content.get( 'file_source', {}).get( 'file_URL', None ) )
            try:
                bname = headers['Content-Disposition']
            except KeyError:
                bname = os.path.basename( urllib2.urlparse.urlsplit( content.get( 'file_source', {}).get( 'file_URL', None ) ).path )
        else:
            filename = content.get( 'file_source', {}).get( 'file_history', None )
            bname = os.path.basename( filename )
        file_action = content.get( 'file_action', {}).get( 'file_action_selector', None )
        if file_action == 'unpack':
            unpack_archive( filename, target_path )
        else:
            filename_override = content.get( 'file_action', {}).get( 'filename_override', None )
            if filename_override:
                target_path = os.path.join( target_path, filename_override )
            else:
                target_path = os.path.join( target_path, bname )
            shutil.copyfile( filename, target_path )
    return len( directory_content )

    def restore(self, backup_file, message=str(), delay=0):
        """
        Restores the backup backup_file.

        Raises: WorldError
        """
        with tempfile.TemporaryDirectory() as temp_dir:
            shutil.unpack_archive(
                filename = backup_file,
                extract_dir = temp_dir
                )
            
            # Stop the world.
            was_online = self.world.is_online()
            if was_online:
                self.world.send_command("say {}".format(message))
                time.sleep(delay)
                self.world.kill_processes()

            # Delete the world and restore the backup.
            for i in range(5):
                # XXX: Fixes an error with server.log.lck
                # and shutil.rmtree(...).
                try:
                    shutil.rmtree(self.world.directory)
                    break
                except OSError:
                    time.sleep(0.05)
            shutil.copytree(temp_dir, self.world.directory)
            
        if was_online:
            self.world.start()
        return None

 def get_cltk_latin_linguistic_data_tar(self):
     """Get CLTK's ML taggers, tokenizers, etc."""
     orig_files_dir_ling_latin = \
         os.path.join(self.orig_files_dir, 'cltk_latin_linguistic_data')
     latin_dir = os.path.join(self.cltk_data, 'latin')
     if os.path.isdir(latin_dir) is True:
         pass
     else:
         os.mkdir(latin_dir)
     latin_dir_ling = os.path.join(latin_dir, 'cltk_linguistic_data')
     if os.path.isdir(latin_dir_ling) is True:
         pass
     else:
         os.mkdir(latin_dir_ling)
     pg_url = 'https://raw.githubusercontent.com/cltk/cltk_latin_linguistic_data/master/latin.tar.gz'
     session = requests.Session()
     session.mount(pg_url, SSLAdapter(ssl.PROTOCOL_TLSv1))
     pg_tar = session.get(pg_url, stream=True)
     ling_latin_file_name = urlsplit(pg_url).path.split('/')[-1]
     tar_latin_file_path = os.path.join(orig_files_dir_ling_latin,
                                        ling_latin_file_name)
     try:
         with open(tar_latin_file_path, 'wb') as new_file:
             new_file.write(pg_tar.content)
             logging.info('Finished writing %s.', ling_latin_file_name)
             try:
                 shutil.unpack_archive(tar_latin_file_path,
                                       latin_dir_ling)
                 logging.info('Finished unpacking %s.',
                              ling_latin_file_name)
             except IOError:
                 logging.info('Failed to unpack %s.',
                              ling_latin_file_name)
     except IOError:
         logging.error('Failed to write file %s', ling_latin_file_name)

    def restore(self, backup_file, message=str(), delay=0):
        """
        Restores the backup of the world from the given *backup_file*. If
        the backup archive contains the server executable it will be restored
        too if necessairy.

        Exceptions:
            * WorldStartFailed
            * WorldStopFailed
            * ... shutil.unpack_archive() exceptions ...
        """
        # Extract the backup in a temporary directory and copy then all things
        # into the EMSM directories.
        with tempfile.TemporaryDirectory() as temp_dir:
            shutil.unpack_archive(
                filename = backup_file,
                extract_dir = temp_dir
                )

            # Stop the world.
            was_online = self._world.is_online()
            if was_online:
                self._world.send_command("say {}".format(message))
                time.sleep(delay)
                self._world.kill_processes()

            # Restore the world.
            self._restore_world(temp_dir)
            self._restore_world_conf(temp_dir)

        # Restart the world if it was online before restoring.
        if was_online:
            self._world.start()
        return None

def import_yaml():
    '''
    import user pre-defined files from ~/xpdUser/Import

    Files can be compreesed or .yml, once imported, bt.list() should show updated acquire object list
    '''
    src_dir = glbl.import_dir
    dst_dir = glbl.yaml_dir
    f_list = os.listdir(src_dir)
    if len(f_list) == 0:
        print('INFO: There is no pre-defined user objects in {}'.format(src_dir))
        return 
    # two possibilites: .yml or compressed files; shutil should handle all compressed cases
    moved_f_list = []
    for f in f_list:
        full_path = os.path.join(src_dir, f)
        (root, ext) = os.path.splitext(f)
        if ext == '.yml':
            shutil.copy(full_path, dst_dir)
            moved_f_list.append(f)
            # FIXME - do we want user confirmation?
            os.remove(full_path)
        else:
            try:
                shutil.unpack_archive(full_path, dst_dir)
                moved_f_list.append(f)
                # FIXME - do we want user confirmation?
                os.remove(full_path)
            except ReadError:
                print('Unrecongnized file type {} is found inside {}'.format(f, src_dir))
                pass
    return moved_f_list