Python s3fs.S3FileSystem方法代码示例

# 需要导入模块: import s3fs [as 别名]
# 或者: from s3fs import S3FileSystem [as 别名]
def get_filepath_or_buffer(filepath_or_buffer, encoding=None,
                           compression=None, mode=None):

    if mode is None:
        mode = 'rb'

    fs = s3fs.S3FileSystem(anon=False)
    try:
        filepath_or_buffer = fs.open(_strip_schema(filepath_or_buffer), mode)
    except (compat.FileNotFoundError, NoCredentialsError):
        # boto3 has troubles when trying to access a public file
        # when credentialed...
        # An OSError is raised if you have credentials, but they
        # aren't valid for that bucket.
        # A NoCredentialsError is raised if you don't have creds
        # for that bucket.
        fs = s3fs.S3FileSystem(anon=True)
        filepath_or_buffer = fs.open(_strip_schema(filepath_or_buffer), mode)
    return filepath_or_buffer, None, compression, True 

# 需要导入模块: import s3fs [as 别名]
# 或者: from s3fs import S3FileSystem [as 别名]
def s3fs_nifti_write(img, fname, fs=None):
    """
    Write a nifti file straight to S3

    Paramters
    ---------
    img : nib.Nifti1Image class instance
        The image containing data to be written into S3
    fname : string
        Full path (including bucket name and extension) to the S3 location
        where the file is to be saved.
    fs : an s3fs.S3FileSystem class instance, optional
        A file-system to refer to. Default to create a new file-system
    """
    if fs is None:
        fs = s3fs.S3FileSystem()

    bio = BytesIO()
    file_map = img.make_file_map({'image': bio, 'header': bio})
    img.to_file_map(file_map)
    data = gzip.compress(bio.getvalue())
    with fs.open(fname, 'wb') as ff:
        ff.write(data) 

# 需要导入模块: import s3fs [as 别名]
# 或者: from s3fs import S3FileSystem [as 别名]
def s3fs_json_read(fname, fs=None):
    """
    Reads json directly from S3

    Paramters
    ---------
    fname : str
        Full path (including bucket name and extension) to the file on S3.
    fs : an s3fs.S3FileSystem class instance, optional
        A file-system to refer to. Default to create a new file-system.

    """
    if fs is None:
        fs = s3fs.S3FileSystem()
    with fs.open(fname) as ff:
        data = json.load(ff)
    return data 

# 需要导入模块: import s3fs [as 别名]
# 或者: from s3fs import S3FileSystem [as 别名]
def s3fs_json_write(data, fname, fs=None):
    """
    Writes json from a dict directly into S3

    Parameters
    ----------
    data : dict
        The json to be written out
    fname : str
        Full path (including bucket name and extension) to the file to
        be written out on S3
    fs : an s3fs.S3FileSystem class instance, optional
        A file-system to refer to. Default to create a new file-system.
    """
    if fs is None:
        fs = s3fs.S3FileSystem()
    with fs.open(fname, 'w') as ff:
        json.dump(data, ff) 

# 需要导入模块: import s3fs [as 别名]
# 或者: from s3fs import S3FileSystem [as 别名]
def test_s3_levels(self):
        with moto.mock_s3():
            self._write_test_cube_pyramid()

            s3 = s3fs.S3FileSystem(key='test_fake_id',
                                   secret='test_fake_secret',
                                   client_kwargs=dict(endpoint_url="https://s3.amazonaws.com"))
            ml_dataset = ObjectStorageMultiLevelDataset(s3,
                                                        "xcube-test/cube-1-250-250.levels",
                                                        chunk_cache_capacity=1000 * 1000 * 1000)
            self.assertIsNotNone(ml_dataset)
            self.assertEqual(3, ml_dataset.num_levels)
            self.assertEqual((250, 250), ml_dataset.tile_grid.tile_size)
            self.assertEqual(2, ml_dataset.tile_grid.num_level_zero_tiles_x)
            self.assertEqual(1, ml_dataset.tile_grid.num_level_zero_tiles_y)
            self.assertEqual(761904762, ml_dataset.get_chunk_cache_capacity(0))
            self.assertEqual(190476190, ml_dataset.get_chunk_cache_capacity(1))
            self.assertEqual(47619048, ml_dataset.get_chunk_cache_capacity(2)) 

# 需要导入模块: import s3fs [as 别名]
# 或者: from s3fs import S3FileSystem [as 别名]
def get_path_or_obs_store(path_or_url: str,
                          client_kwargs: Mapping[str, Any] = None,
                          mode: str = 'r') -> Tuple[Union[str, Dict], bool]:
    """
    If *path_or_url* is an object storage URL, return a object storage Zarr store (mapping object)
    using *client_kwargs* and *mode* and a flag indicating whether the Zarr datasets is consolidated.

    Otherwise *path_or_url* is interpreted as a local file system path, retured as-is plus
    a flag indicating whether the Zarr datasets is consolidated.

    :param path_or_url: A path or a URL.
    :param client_kwargs: Object storage client keyword arguments.
    :param mode: "r" or "w"
    :return: A tuple (path_or_obs_store, consolidated).
    """
    if is_obs_url(path_or_url):
        root, obs_fs_kwargs, obs_fs_client_kwargs = parse_obs_url_and_kwargs(path_or_url, client_kwargs)
        s3 = s3fs.S3FileSystem(**obs_fs_kwargs, client_kwargs=obs_fs_client_kwargs)
        consolidated = mode == "r" and s3.exists(f'{root}/.zmetadata')
        return s3fs.S3Map(root=root, s3=s3, check=False, create=mode == "w"), consolidated
    else:
        consolidated = os.path.exists(os.path.join(path_or_url, '.zmetadata'))
        return path_or_url, consolidated 

# 需要导入模块: import s3fs [as 别名]
# 或者: from s3fs import S3FileSystem [as 别名]
def __init__(
        self,
        *,
        path,
        batch_size=None,
        columns=None,
        batch_aggregator=1
    ):
        super().__init__()
        self.path = path
        self.batch_size = batch_size
        self.columns = columns
        self.batch_aggregator = batch_aggregator

        if self.path.startswith('s3://'):
            s3 = s3fs.S3FileSystem()
            self.files = [f's3://{file}' for file in s3.ls(self.path)]
        else:
            self.files = [
                os.path.join(path, file)
                for file in os.listdir(self.path)
            ]
        self.files = [f for f in self.files if f.endswith('.parquet')] 

# 需要导入模块: import s3fs [as 别名]
# 或者: from s3fs import S3FileSystem [as 别名]
def file_exists(cls, file_path):
        if isinstance(file_path, str):
            match = S3_ADDRESS_REGEX.search(file_path)
            if match is not None:
                if file_path[0] == "S":
                    file_path = "{}{}".format("s", file_path[1:])
                import s3fs as S3FS
                from botocore.exceptions import NoCredentialsError

                s3fs = S3FS.S3FileSystem(anon=False)
                exists = False
                try:
                    exists = s3fs.exists(file_path) or exists
                except NoCredentialsError:
                    pass
                s3fs = S3FS.S3FileSystem(anon=True)
                return exists or s3fs.exists(file_path)
        return os.path.exists(file_path) 

# 需要导入模块: import s3fs [as 别名]
# 或者: from s3fs import S3FileSystem [as 别名]
def get_fs(self):
        return s3fs.S3FileSystem(anon=False, default_block_size=SPECTRIFY_BLOCKSIZE) 

# 需要导入模块: import s3fs [as 别名]
# 或者: from s3fs import S3FileSystem [as 别名]
def s3fs_nifti_read(fname, fs=None):
    """
    Lazily reads a nifti image from S3.

    Paramters
    ---------
    fname : string
        Full path (including bucket name and extension) to the S3 location
        of the file to be read.
    fs : an s3fs.S3FileSystem class instance, optional
        A file-system to refer to. Default to create a new file-system.

    Returns
    -------
    nib.Nifti1Image class instance

    Note
    ----
    Because the image is lazily loaded, data stored in the file
    is not transferred until `get_fdata` is called.

    """
    if fs is None:
        fs = s3fs.S3FileSystem()
    with fs.open(fname) as ff:
        zz = gzip.open(ff)
        rr = zz.read()
        bb = BytesIO(rr)
        fh = nib.FileHolder(fileobj=bb)
        img = nib.Nifti1Image.from_file_map({'header': fh, 'image': fh})
    return img 

# 需要导入模块: import s3fs [as 别名]
# 或者: from s3fs import S3FileSystem [as 别名]
def setUp(self):
        self.s3 = S3FileSystem(anon=True, use_ssl=False)
        self.key = '/'.join([
            'common-crawl',
            'crawl-data',
            'CC-MAIN-2016-07',
            'segments',
            '1454702039825.90',
            'warc',
            'CC-MAIN-20160205195359-00348-ip-10-236-182-209.ec2.internal.warc.gz',
        ])
        self.s3_url = 's3://aws-publicdatasets/{key}'.format(key=self.key) 

# 需要导入模块: import s3fs [as 别名]
# 或者: from s3fs import S3FileSystem [as 别名]
def test_remote(self):
        import s3fs
        endpoint_url = "http://obs.eu-de.otc.t-systems.com"
        s3 = s3fs.S3FileSystem(anon=True, client_kwargs=dict(endpoint_url=endpoint_url))
        s3_store = s3fs.S3Map(root="cyanoalert/cyanoalert-olci-lswe-l2c-v1.zarr", s3=s3, check=False)
        diagnostic_store = DiagnosticStore(s3_store, logging_observer(log_path='remote-cube.log'))
        xr.open_zarr(diagnostic_store) 

# 需要导入模块: import s3fs [as 别名]
# 或者: from s3fs import S3FileSystem [as 别名]
def parse_obs_url_and_kwargs(obs_url: str, obs_kwargs: Mapping[str, Any]) -> Tuple[str, Dict[str, Any], Dict[str, Any]]:
    """
    Parses *obs_url* and *kwargs* and returns a
    tuple (*root*, *kwargs*, *client_kwargs*) whose elements
    can be passed to the s3fs.S3FileSystem and s3fs.S3Map constructors as follows:::

        obs_fs = s3fs.S3FileSystem(**kwargs, client_kwargs=client_kwargs)
        obs_map = s3fs.S3Map(root=root, s3=obs_fs)

    :param obs_url: Object storage URL, e.g. "s3://bucket/root", or "https://bucket.s3.amazonaws.com/root".
    :param obs_kwargs: Keyword arguments.
    :return: A tuple (root, kwargs, client_kwargs).
    """

    anon = True
    key = None
    secret = None
    client_kwargs = dict(obs_kwargs) if obs_kwargs else dict()

    endpoint_url, root = split_obs_url(obs_url)
    if endpoint_url:
        client_kwargs['endpoint_url'] = endpoint_url

    if 'provider_access_key_id' in client_kwargs:
        key = client_kwargs.pop('provider_access_key_id')
    if 'aws_access_key_id' in client_kwargs:
        key = client_kwargs.pop('aws_access_key_id')
    if 'provider_secret_access_key' in client_kwargs:
        secret = client_kwargs.pop('provider_secret_access_key')
    if 'aws_secret_access_key' in client_kwargs:
        secret = client_kwargs.pop('aws_secret_access_key')
    if key and secret:
        anon = False
    else:
        key = secret = None

    return root, dict(anon=anon, key=key, secret=secret), client_kwargs 

# 需要导入模块: import s3fs [as 别名]
# 或者: from s3fs import S3FileSystem [as 别名]
def __init__(self, s3_fs: s3fs.S3FileSystem = None):
        self._s3_fs = s3_fs

    # noinspection PyUnusedLocal,PyMethodMayBeStatic 

# 需要导入模块: import s3fs [as 别名]
# 或者: from s3fs import S3FileSystem [as 别名]
def open_ml_dataset_from_object_storage(path: str,
                                        data_format: str = None,
                                        ds_id: str = None,
                                        exception_type: type = ValueError,
                                        client_kwargs: Mapping[str, Any] = None,
                                        chunk_cache_capacity: int = None,
                                        **kwargs) -> MultiLevelDataset:
    data_format = data_format or guess_ml_dataset_format(path)

    root, obs_fs_kwargs, obs_fs_client_kwargs = parse_obs_url_and_kwargs(path, client_kwargs)
    obs_fs = s3fs.S3FileSystem(**obs_fs_kwargs, client_kwargs=obs_fs_client_kwargs)

    if data_format == FORMAT_NAME_ZARR:
        store = s3fs.S3Map(root=root, s3=obs_fs, check=False)
        if chunk_cache_capacity:
            store = zarr.LRUStoreCache(store, max_size=chunk_cache_capacity)
        with measure_time(tag=f"opened remote zarr dataset {path}"):
            consolidated = obs_fs.exists(f'{root}/.zmetadata')
            ds = assert_cube(xr.open_zarr(store, consolidated=consolidated, **kwargs))
        return BaseMultiLevelDataset(ds, ds_id=ds_id)
    elif data_format == FORMAT_NAME_LEVELS:
        with measure_time(tag=f"opened remote levels dataset {path}"):
            return ObjectStorageMultiLevelDataset(obs_fs,
                                                  root,
                                                  zarr_kwargs=kwargs,
                                                  ds_id=ds_id,
                                                  chunk_cache_capacity=chunk_cache_capacity,
                                                  exception_type=exception_type)

    raise exception_type(f'Unrecognized multi-level dataset format {data_format!r} for path {path!r}')