Python boto.s3方法代码示例

# 需要导入模块: import boto [as 别名]
# 或者: from boto import s3 [as 别名]
def get_logging_status(self, headers=None):
        """
        Get the logging status for this bucket.

        :rtype: :class:`boto.s3.bucketlogging.BucketLogging`
        :return: A BucketLogging object for this bucket.
        """
        response = self.connection.make_request('GET', self.name,
                query_args='logging', headers=headers)
        body = response.read()
        if response.status == 200:
            blogging = BucketLogging()
            h = handler.XmlHandler(blogging, self)
            if not isinstance(body, bytes):
                body = body.encode('utf-8')
            xml.sax.parseString(body, h)
            return blogging
        else:
            raise self.connection.provider.storage_response_error(
                response.status, response.reason, body) 

# 需要导入模块: import boto [as 别名]
# 或者: from boto import s3 [as 别名]
def load_from_s3_file(s3_uri):
    """Load data from S3
    Useful for loading small config or schema files

    :param s3_uri: path to S3 uri
    :returns: file contents
    """
    _, _, path = s3_uri.partition('://')
    bucket_name, _, key_name = path.partition('/')

    # if region is in a bucket name, put that region first
    def preferred_region(item):
        return item.name not in bucket_name

    boto_creds = get_boto_creds()
    for region in sorted(boto.s3.regions(), key=preferred_region):
        try:
            conn = boto.s3.connect_to_region(region.name, **boto_creds)
            return _load_from_s3_region(conn, bucket_name, key_name)
        except S3ResponseError as e:
            # skip to next region if access is not allowed from this one
            if e.status not in [403, 301]:
                raise
    raise ValueError("{0}: No valid region found".format(s3_uri)) 

# 需要导入模块: import boto [as 别名]
# 或者: from boto import s3 [as 别名]
def get_lifecycle_config(self, headers=None):
        """
        Returns the current lifecycle configuration on the bucket.

        :rtype: :class:`boto.s3.lifecycle.Lifecycle`
        :returns: A LifecycleConfig object that describes all current
            lifecycle rules in effect for the bucket.
        """
        response = self.connection.make_request('GET', self.name,
                query_args='lifecycle', headers=headers)
        body = response.read()
        boto.log.debug(body)
        if response.status == 200:
            lifecycle = Lifecycle()
            h = handler.XmlHandler(lifecycle, self)
            if not isinstance(body, bytes):
                body = body.encode('utf-8')
            xml.sax.parseString(body, h)
            return lifecycle
        else:
            raise self.connection.provider.storage_response_error(
                response.status, response.reason, body) 

# 需要导入模块: import boto [as 别名]
# 或者: from boto import s3 [as 别名]
def printUsage():
    print("usage: python get_s3json [--bucket_name=<bucket>] [--aws_s3_gateway=<s3_endpoint>] objid ")
    print("  objid: s3 JSON obj to fetch")
    print("  Example: python get_s3json --aws_s3_gateway=http://192.168.99.100:9000 --bucket_name=minio.hsdsdev t-cf2fc310-996f-11e6-8ef6-0242ac110005")
    sys.exit(); 

#
# Get hash prefix
# 

# 需要导入模块: import boto [as 别名]
# 或者: from boto import s3 [as 别名]
def main():
    if len(sys.argv) == 1 or sys.argv[1] == "-h" or sys.argv[1] == "--help":
        printUsage()
        sys.exit(1)
    
    obj_id = sys.argv[-1]
    s3_gateway = config.get("aws_s3_gateway")
    print("aws_s3_gateway: {}".format(s3_gateway))
    region = config.get("aws_region")
    print("region: {}".format(region))
    print("now: {}".format(time.time()))
    conn = boto.s3.connect_to_region(
        region,
        calling_format=boto.s3.connection.OrdinaryCallingFormat()
    )
    
    bucket_name = config.get("bucket_name")
    print("bucket_name: {}".format(bucket_name))
    bucket = conn.get_bucket(bucket_name)

    if obj_id.startswith("d-") or obj_id.startswith("g-") or obj_id.startswith("t-"):
        # add the checksum prefix
        obj_id = getIdHash(obj_id) + '-' + obj_id

    k = Key(bucket)
    k.key = obj_id
    data = k.get_contents_as_string()
    if not isinstance(data, str):
        # Python 3 - convert from bytes to str
        data = data.decode("utf-8")
    json_data = json.loads(data)
    print(json.dumps(json_data, sort_keys=True, indent=4)) 

# 需要导入模块: import boto [as 别名]
# 或者: from boto import s3 [as 别名]
def is_s3_uri(uri):
    uri = str(uri)
    return uri.startswith('s3://') or uri.startswith('s3n://') 

# 需要导入模块: import boto [as 别名]
# 或者: from boto import s3 [as 别名]
def __init__(self, region='us-east-1'):
        # use a single provider to avoid NoAuthHandler exceptions
        # see: http://blog.johnryding.com/post/122337566993/solving-intermittent-noauthhandlerfound-errors-in
        if S3.aws_settings_provider is None:
            S3.aws_settings_provider = boto.provider.Provider('aws')

        self._conn = boto.s3.connect_to_region(
            region,
            calling_format=boto.s3.connection.OrdinaryCallingFormat(),
            provider=S3.aws_settings_provider
        ) 

# 需要导入模块: import boto [as 别名]
# 或者: from boto import s3 [as 别名]
def _list_prefix(self, s3uri):
        results = self._conn.get_bucket(s3uri.bucket).list(s3uri.path, delimiter='/')
        return (S3Uri('s3://{0}/{1}'.format(s3uri.bucket, i.name)) for i in results) 

# 需要导入模块: import boto [as 别名]
# 或者: from boto import s3 [as 别名]
def _list_buckets(self):
        return (S3Uri('s3://{0}'.format(i.name)) for i in self._conn.get_all_buckets()) 

# 需要导入模块: import boto [as 别名]
# 或者: from boto import s3 [as 别名]
def rm(self, uris):
        uris = [S3Uri(uri) for uri in uris]

        for bucket, group in groupby(
            sorted(uris, key=lambda uri: uri.bucket), lambda i: i.bucket):
                returned_keys = self._conn.get_bucket(bucket)\
                                    .delete_keys(
                                        boto.s3.key.Key(bucket, i.path) for i in group)

                if(len(returned_keys.errors) > 0):
                    raise IOError('Could not delete keys: {keys}'.format(
                        keys=[k for k in returned_keys.errors])) 

# 需要导入模块: import boto [as 别名]
# 或者: from boto import s3 [as 别名]
def __init_s3(cls):
        if not cls.s3:
            cls.s3 = S3() 

# 需要导入模块: import boto [as 别名]
# 或者: from boto import s3 [as 别名]
def close(self):
        if 'w' in self.mode:
            self.flush()
            self.s3.putfile(self, self.s3uri) 

# 需要导入模块: import boto [as 别名]
# 或者: from boto import s3 [as 别名]
def connect(self):
        if not self._conn:
            try:
                if (self.access_key is not None and self.secret_key is not None):
                    logging.debug("Connecting to AWS S3 with Access Key: %s" % self.access_key)
                    self._conn = boto.s3.connect_to_region(
                        self.region,
                        aws_access_key_id=self.access_key,
                        aws_secret_access_key=self.secret_key,
                        is_secure=self.secure,
                        calling_format=self.calling_format
                    )
                    logging.debug("Successfully connected to AWS S3 with Access Key: %s" % self.access_key)
                else:
                    logging.debug("Connecting to AWS S3 with IAM Role")
                    self._conn = boto.s3.connect_to_region(
                        self.region,
                        is_secure=self.secure,
                        calling_format=self.calling_format
                    )
                    logging.debug("Successfully connected to AWS S3 with IAM Role")
            except boto.exception.S3ResponseError, e:
                if self.is_forbidden_error(e):
                    logging.error("Not authorized to connect to AWS S3 with Access Key: %s!" % self.access_key)
                else:
                    logging.error("Cannot connect to AWS S3 with Access Key: %s!" % self.access_key)
                return OperationError(e)
            except Exception, e:
                logging.error("Cannot connect to AWS S3 with Access Key: %s!" % self.access_key)
                raise OperationError(e) 

# 需要导入模块: import boto [as 别名]
# 或者: from boto import s3 [as 别名]
def is_s3_path(file_path):
    """ Return true if file_path is an S3 path, else false.
    """
    schema, _, rest = file_path.partition('://')
    return schema == 's3' 

# 需要导入模块: import boto [as 别名]
# 或者: from boto import s3 [as 别名]
def parse_s3_path(file_path):
    if not is_s3_path(file_path):
        raise ValueError('{0} is not a valid s3 path'.format(file_path))
    parse_array = file_path.split("/", 3)
    bucket = parse_array[2]
    prefix = parse_array[3]
    return bucket, prefix