Python gcsfs.GCSFileSystem方法代码示例

# 需要导入模块: import gcsfs [as 别名]
# 或者: from gcsfs import GCSFileSystem [as 别名]
def __init__(self, config,  bucket = None, executor_id = None):
        logger.debug("Creating gcsfs storage client")
        self.config = config
        self.bucket = bucket
        self.fs = gcsfs.GCSFileSystem(project=config["project_id"])
        logger.debug("gcsfs storage client created successfully") 

# 需要导入模块: import gcsfs [as 别名]
# 或者: from gcsfs import GCSFileSystem [as 别名]
def get_filepath_or_buffer(filepath_or_buffer, encoding=None,
                           compression=None, mode=None):

    if mode is None:
        mode = 'rb'

    fs = gcsfs.GCSFileSystem()
    filepath_or_buffer = fs.open(filepath_or_buffer, mode)
    return filepath_or_buffer, None, compression, True 

# 需要导入模块: import gcsfs [as 别名]
# 或者: from gcsfs import GCSFileSystem [as 别名]
def gcs_agnostic_open(fn, mode='r'):
    if fn.startswith('gs://'):
        return gcsfs.GCSFileSystem().open(fn, mode)
    else:
        return open(fn, mode) 

# 需要导入模块: import gcsfs [as 别名]
# 或者: from gcsfs import GCSFileSystem [as 别名]
def download_result(
    result_file
):
    import gcsfs
    fs = gcsfs.GCSFileSystem()
    fs.get(result_file, 'submission.csv') 

# 需要导入模块: import gcsfs [as 别名]
# 或者: from gcsfs import GCSFileSystem [as 别名]
def __init__(self, log, **kwargs):
        super(GCSFS, self).__init__(**kwargs)
        self.log = log

        token = os.path.expanduser(self.token)
        self.fs = gcsfs.GCSFileSystem(project=self.project, token=token)

        self.init() 

# 需要导入模块: import gcsfs [as 别名]
# 或者: from gcsfs import GCSFileSystem [as 别名]
def open_local_or_gs(path, mode):
    open_func = gcsfs.GCSFileSystem().open if is_gs_path(path) else open
    return open_func(path, mode) 

# 需要导入模块: import gcsfs [as 别名]
# 或者: from gcsfs import GCSFileSystem [as 别名]
def get_bucket_files(glob_pattern, base_dir, force=False, pattern_slice=slice(None)):
    """Helper function to download files from Google Cloud Storage.

    Args:
        glob_pattern (str or list): Glob pattern string or series of patterns
            used to search for on Google Cloud Storage. The pattern should
            include the "gs://" protocol prefix. If a list of lists, then the
            results of each sublist pattern are concatenated and the result is
            treated as one pattern result. This is important for things like
            ``pattern_slice`` and complicated glob patterns not supported by
            GCP.
        base_dir (str): Root directory to place downloaded files on the local
            system.
        force (bool): Force re-download of data regardless of its existence on
            the local system. Warning: May delete non-demo files stored in
            download directory.
        pattern_slice (slice): Slice object to limit the number of files
            returned by each glob pattern.

    """
    if gcsfs is None:
        raise RuntimeError("Missing 'gcsfs' dependency for GCS download.")
    if not os.path.isdir(base_dir):
        # it is the caller's responsibility to make this
        raise OSError("Directory does not exist: {}".format(base_dir))

    if isinstance(glob_pattern, str):
        glob_pattern = [glob_pattern]

    fs = gcsfs.GCSFileSystem(token='anon')
    filenames = []
    for gp in glob_pattern:
        # handle multiple glob patterns being treated as one pattern
        # for complicated patterns that GCP can't handle
        if isinstance(gp, str):
            glob_results = list(fs.glob(gp))
        else:
            # flat list of results
            glob_results = [fn for pat in gp for fn in fs.glob(pat)]

        for fn in glob_results[pattern_slice]:
            ondisk_fn = os.path.basename(fn)
            ondisk_pathname = os.path.join(base_dir, ondisk_fn)
            filenames.append(ondisk_pathname)

            if force and os.path.isfile(ondisk_pathname):
                os.remove(ondisk_pathname)
            elif os.path.isfile(ondisk_pathname):
                LOG.info("Found existing: {}".format(ondisk_pathname))
                continue
            LOG.info("Downloading: {}".format(ondisk_pathname))
            fs.get('gs://' + fn, ondisk_pathname)

    if not filenames:
        raise OSError("No files could be found or downloaded.")
    return filenames