Golang s3.S3类代码示例

func (g *S3Getter) getObject(client *s3.S3, dst, bucket, key, version string) error {
	req := &s3.GetObjectInput{
		Bucket: aws.String(bucket),
		Key:    aws.String(key),
	}
	if version != "" {
		req.VersionId = aws.String(version)
	}

	resp, err := client.GetObject(req)
	if err != nil {
		return err
	}

	// Create all the parent directories
	if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil {
		return err
	}

	f, err := os.Create(dst)
	if err != nil {
		return err
	}
	defer f.Close()

	_, err = io.Copy(f, resp.Body)
	return err
}

func resourceAwsS3BucketLoggingUpdate(s3conn *s3.S3, d *schema.ResourceData) error {
	logging := d.Get("logging").(*schema.Set).List()
	bucket := d.Get("bucket").(string)
	loggingStatus := &s3.BucketLoggingStatus{}

	if len(logging) > 0 {
		c := logging[0].(map[string]interface{})

		loggingEnabled := &s3.LoggingEnabled{}
		if val, ok := c["target_bucket"]; ok {
			loggingEnabled.TargetBucket = aws.String(val.(string))
		}
		if val, ok := c["target_prefix"]; ok {
			loggingEnabled.TargetPrefix = aws.String(val.(string))
		}

		loggingStatus.LoggingEnabled = loggingEnabled
	}

	i := &s3.PutBucketLoggingInput{
		Bucket:              aws.String(bucket),
		BucketLoggingStatus: loggingStatus,
	}
	log.Printf("[DEBUG] S3 put bucket logging: %#v", i)

	_, err := s3conn.PutBucketLogging(i)
	if err != nil {
		return fmt.Errorf("Error putting S3 logging: %s", err)
	}

	return nil
}

func (s *S3) UploadFileD(S3 *s3.S3, key string, file *os.File, contentType, perm string) (*s3.PutObjectInput, *s3.PutObjectOutput, error) {

	md5hex, _, _, err := Md5Sum(file)
	if err != nil {
		return nil, nil, err
	}
	//s.Log.Debugf("key:%s md5=%s", key, md5hex)
	req := &s3.PutObjectInput{
		ACL:    &perm,
		Body:   file,
		Bucket: &s.Bucket,
		//ContentLength: &size,
		ContentType: &contentType,
		Key:         &key,
	}
	res, err := S3.PutObject(req)
	if err != nil {
		return req, res, err
	}
	if res == nil {
		return req, res, fmt.Errorf("res is nil pointer")
	}
	if res.ETag == nil {
		return req, res, fmt.Errorf("res.ETag is nil pointer")
	}
	if len(*res.ETag) < 2 {
		return req, res, fmt.Errorf("*res.ETag is too short. It should have 2 characters or more")
	}
	etag := (*res.ETag)[1 : len(*res.ETag)-1]
	if md5hex != etag {
		return req, res, fmt.Errorf("md5 and ETag does not match. md5:%s ETag:%s", md5hex, etag)
	}
	return req, res, err
}

// GetS3Object returns the object output for the given object key
func GetS3Object(client *s3.S3, bucket, objKey string) (*s3.GetObjectOutput, error) {
	logger.Debug("GetS3Object: ", bucket, "-", objKey)
	params := &s3.GetObjectInput{
		Bucket: aws.String(bucket), // Required
		Key:    aws.String(objKey), // Required
		// IfMatch:                    aws.String("IfMatch"),
		// IfModifiedSince:            aws.Time(time.Now()),
		// IfNoneMatch:                aws.String("IfNoneMatch"),
		// IfUnmodifiedSince:          aws.Time(time.Now()),
		// Range:                      aws.String("Range"),
		// RequestPayer:               aws.String("RequestPayer"),
		// ResponseCacheControl:       aws.String("ResponseCacheControl"),
		// ResponseContentDisposition: aws.String("ResponseContentDisposition"),
		// ResponseContentEncoding:    aws.String("ResponseContentEncoding"),
		// ResponseContentLanguage:    aws.String("ResponseContentLanguage"),
		// ResponseContentType:        aws.String("ResponseContentType"),
		// ResponseExpires:            aws.Time(time.Now()),
		// SSECustomerAlgorithm:       aws.String("SSECustomerAlgorithm"),
		// SSECustomerKey:             aws.String("SSECustomerKey"),
		// SSECustomerKeyMD5:          aws.String("SSECustomerKeyMD5"),
		// VersionId:                  aws.String("ObjectVersionId"),
	}

	return client.GetObject(params)
}

func deleteObjects(bucket string, objs []Obj, wg *sync.WaitGroup, S3 *s3.S3) {
	maxLen := 1000

	for i := 0; i < len(objs); i += maxLen {
		high := i + maxLen
		if high > len(objs) {
			high = len(objs)
		}

		objects := []*s3.ObjectIdentifier{}
		for _, obj := range objs[i:high] {
			objects = append(objects, &s3.ObjectIdentifier{Key: aws.String(obj.key), VersionId: aws.String(obj.id)})
		}

		_, err := S3.DeleteObjects(&s3.DeleteObjectsInput{
			Bucket: aws.String(bucket),
			Delete: &s3.Delete{
				Objects: objects,
			},
		})
		if err != nil {
			fmt.Printf("Failed: %s\n", err)
		}

		wg.Add(-len(objects))
	}

	return
}

// setTags is a helper to set the tags for a resource. It expects the
// tags field to be named "tags"
func setTagsS3(conn *s3.S3, d *schema.ResourceData) error {
	if d.HasChange("tags") {
		oraw, nraw := d.GetChange("tags")
		o := oraw.(map[string]interface{})
		n := nraw.(map[string]interface{})
		create, remove := diffTagsS3(tagsFromMapS3(o), tagsFromMapS3(n))

		// Set tags
		if len(remove) > 0 {
			log.Printf("[DEBUG] Removing tags: %#v", remove)
			_, err := conn.DeleteBucketTagging(&s3.DeleteBucketTaggingInput{
				Bucket: aws.String(d.Get("bucket").(string)),
			})
			if err != nil {
				return err
			}
		}
		if len(create) > 0 {
			log.Printf("[DEBUG] Creating tags: %#v", create)
			req := &s3.PutBucketTaggingInput{
				Bucket: aws.String(d.Get("bucket").(string)),
				Tagging: &s3.Tagging{
					TagSet: create,
				},
			}

			_, err := conn.PutBucketTagging(req)
			if err != nil {
				return err
			}
		}
	}

	return nil
}

func S3ListObjectOutput(cli *s3.S3, bucket string) (listObjectOutput *s3.ListObjectsOutput, err error) {
	listObjectOutput, err = cli.ListObjects(&s3.ListObjectsInput{
		Bucket: aws.String(bucket),
	})

	return listObjectOutput, err
}

func pull(s3serv *s3.S3, c config, p string) {
	if list, e := s3serv.ListObjects(&s3.ListObjectsInput{Bucket: &c.Bucket}); e == nil {
		for _, l := range list.Contents {
			log.Println(*l.Key)
			if obj, e := s3serv.GetObject(&s3.GetObjectInput{Bucket: &c.Bucket, Key: l.Key}); e == nil {
				os.MkdirAll(path.Dir(p+*l.Key), 0700)
				if f, e := os.OpenFile(p+*l.Key, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0666); e == nil {
					defer f.Close()
					defer obj.Body.Close()
					counter := &CountReader{input: obj.Body, co: c}
					counter.Init()
					io.Copy(f, counter)
					log.Println("written", *l.Key)
				} else {
					log.Println(e)
				}

			} else {
				log.Println(e)
			}
		}
	} else {
		log.Fatal(e)
	}

}

func listBucketObjects(svc *s3.S3, bucket string) ([]Object, []ErrObject, error) {
	listRes, err := svc.ListObjects(&s3.ListObjectsInput{
		Bucket: &bucket,
	})
	if err != nil {
		return nil, nil, err
	}

	objs := make([]Object, 0, len(listRes.Contents))
	errObjs := []ErrObject{}
	for _, listObj := range listRes.Contents {
		objData, err := svc.HeadObject(&s3.HeadObjectInput{
			Bucket: &bucket,
			Key:    listObj.Key,
		})

		if err != nil {
			errObjs = append(errObjs, ErrObject{Bucket: bucket, Key: *listObj.Key, Error: err})
			continue
		}

		obj := Object{Bucket: bucket, Key: *listObj.Key}
		if objData.ServerSideEncryption != nil {
			obj.Encrypted = true
			obj.EncryptionType = *objData.ServerSideEncryption
		}

		objs = append(objs, obj)
	}

	return objs, errObjs, nil
}

func recvUpload(s3serv *s3.S3, c config, files chan string) {

	for {
		path, closed := <-files

		if !closed {
			break
		}

		if f, err := os.Open(path); err == nil {
			counter := &CountReader{input: f, seeker: f, co: c}
			counter.Init()
			fmt.Print("uploading:", path)
			if _, err := s3serv.PutObject(&s3.PutObjectInput{Bucket: &c.Bucket, Key: &path, Body: counter}); err == nil {
				fmt.Println("done", counter.count)

			} else {
				fmt.Println("error uploading file:", err)
			}

		} else {
			fmt.Printf("error opening file:", err)
		}
	}
}

func GetRelatedProductsFromS3(svc *s3.S3, dataDir string) RelatedProducts {
	results := RelatedProducts{Relates: make(map[string]Product)}
	bucket, keypattern := parseS3Params(dataDir)
	params := &s3.ListObjectsInput{
		Bucket: aws.String("ecomm-order-items"), // Required
		//Delimiter:    aws.String("Delimiter"),
		//EncodingType: aws.String("EncodingType"),
		//Marker:       aws.String("Marker"),
		//MaxKeys:      aws.Int64(1),
		//Prefix:       aws.String("Prefix"),
	}
	resp, err := svc.ListObjects(params)

	if err != nil {
		// Print the error, cast err to awserr.Error to get the Code and
		// Message from an error.

		glog.Errorf("%s, %s \n", err.(awserr.Error).Code(), err.(awserr.Error).Error())

	}

	// Pretty-print the response data.
	//fmt.Println(resp)
	for _, obj := range resp.Contents {
		glog.V(2).Infof("s3 object: %s. Keypattern:%s", *obj.Key, keypattern)
		if strings.HasPrefix(*obj.Key, keypattern[1:]+"/part-") {
			glog.V(2).Infof("populating with file %s", *obj.Key)
			populateRelatedProducts(&results, getObject(svc, bucket, *obj.Key))
		}
	}

	return results

}

func resourceAwsS3BucketVersioningUpdate(s3conn *s3.S3, d *schema.ResourceData) error {
	v := d.Get("versioning").(*schema.Set).List()
	bucket := d.Get("bucket").(string)
	vc := &s3.VersioningConfiguration{}

	if len(v) > 0 {
		c := v[0].(map[string]interface{})

		if c["enabled"].(bool) {
			vc.Status = aws.String(s3.BucketVersioningStatusEnabled)
		} else {
			vc.Status = aws.String(s3.BucketVersioningStatusSuspended)
		}
	} else {
		vc.Status = aws.String(s3.BucketVersioningStatusSuspended)
	}

	i := &s3.PutBucketVersioningInput{
		Bucket:                  aws.String(bucket),
		VersioningConfiguration: vc,
	}
	log.Printf("[DEBUG] S3 put bucket versioning: %#v", i)

	_, err := s3conn.PutBucketVersioning(i)
	if err != nil {
		return fmt.Errorf("Error putting S3 versioning: %s", err)
	}

	return nil
}

func head(service *s3.S3, bucketName string, key string) bool {
	_, err := service.HeadObject(&s3.HeadObjectInput{
		Bucket: stringPtr(bucketName),
		Key:    stringPtr(key),
	})
	return err == nil
}

func uploadFile(file string, conf config, svc *s3.S3, results chan<- result) {
	obj, err := newObjectInfo(file)
	if err != nil {
		results <- result{err: fmt.Errorf("unable to read file: %v", err)}
		return
	}

	bname := conf.Buckets[bucketName].Name
	params := &s3.PutObjectInput{
		Bucket:      &bname,
		Key:         &obj.key,
		Body:        obj.body,
		ContentType: &obj.contentType,
	}

	if !private {
		params.ACL = aws.String("public-read")
	}

	_, err = svc.PutObject(params)

	if err != nil {
		results <- result{err: err}
	} else {
		results <- result{url: buildOutputURL(obj, conf)}
	}
}

func ListObjects(svc *s3.S3) {

	//svc := s3.New(session.New(),&aws.Config{Region: aws.String("us-east-1")})

	params := &s3.ListObjectsInput{
		Bucket: aws.String("ecomm-order-items"), // Required
		//Delimiter:    aws.String("Delimiter"),
		//EncodingType: aws.String("EncodingType"),
		//Marker:       aws.String("Marker"),
		//MaxKeys:      aws.Int64(1),
		//Prefix:       aws.String("Prefix"),
	}
	resp, err := svc.ListObjects(params)

	if err != nil {
		// Print the error, cast err to awserr.Error to get the Code and
		// Message from an error.

		fmt.Printf("%s, %s \n", err.(awserr.Error).Code(), err.(awserr.Error).Error())
		return
	}

	// Pretty-print the response data.
	//fmt.Println(resp)
	for _, obj := range resp.Contents {
		if strings.HasPrefix(*obj.Key, "recommendations/output.txt/part-") {
			fmt.Println(GetObject(svc, *obj.Key))
		}
	}
}