Golang S3.ListObjects方法代碼示例

func listBucketObjects(svc *s3.S3, bucket string) ([]Object, []ErrObject, error) {
	listRes, err := svc.ListObjects(&s3.ListObjectsInput{
		Bucket: &bucket,
	})
	if err != nil {
		return nil, nil, err
	}

	objs := make([]Object, 0, len(listRes.Contents))
	errObjs := []ErrObject{}
	for _, listObj := range listRes.Contents {
		objData, err := svc.HeadObject(&s3.HeadObjectInput{
			Bucket: &bucket,
			Key:    listObj.Key,
		})

		if err != nil {
			errObjs = append(errObjs, ErrObject{Bucket: bucket, Key: *listObj.Key, Error: err})
			continue
		}

		obj := Object{Bucket: bucket, Key: *listObj.Key}
		if objData.ServerSideEncryption != nil {
			obj.Encrypted = true
			obj.EncryptionType = *objData.ServerSideEncryption
		}

		objs = append(objs, obj)
	}

	return objs, errObjs, nil
}

func GetRelatedProductsFromS3(svc *s3.S3, dataDir string) RelatedProducts {
	results := RelatedProducts{Relates: make(map[string]Product)}
	bucket, keypattern := parseS3Params(dataDir)
	params := &s3.ListObjectsInput{
		Bucket: aws.String("ecomm-order-items"), // Required
		//Delimiter:    aws.String("Delimiter"),
		//EncodingType: aws.String("EncodingType"),
		//Marker:       aws.String("Marker"),
		//MaxKeys:      aws.Int64(1),
		//Prefix:       aws.String("Prefix"),
	}
	resp, err := svc.ListObjects(params)

	if err != nil {
		// Print the error, cast err to awserr.Error to get the Code and
		// Message from an error.

		glog.Errorf("%s, %s \n", err.(awserr.Error).Code(), err.(awserr.Error).Error())

	}

	// Pretty-print the response data.
	//fmt.Println(resp)
	for _, obj := range resp.Contents {
		glog.V(2).Infof("s3 object: %s. Keypattern:%s", *obj.Key, keypattern)
		if strings.HasPrefix(*obj.Key, keypattern[1:]+"/part-") {
			glog.V(2).Infof("populating with file %s", *obj.Key)
			populateRelatedProducts(&results, getObject(svc, bucket, *obj.Key))
		}
	}

	return results

}

func ListObjects(svc *s3.S3) {

	//svc := s3.New(session.New(),&aws.Config{Region: aws.String("us-east-1")})

	params := &s3.ListObjectsInput{
		Bucket: aws.String("ecomm-order-items"), // Required
		//Delimiter:    aws.String("Delimiter"),
		//EncodingType: aws.String("EncodingType"),
		//Marker:       aws.String("Marker"),
		//MaxKeys:      aws.Int64(1),
		//Prefix:       aws.String("Prefix"),
	}
	resp, err := svc.ListObjects(params)

	if err != nil {
		// Print the error, cast err to awserr.Error to get the Code and
		// Message from an error.

		fmt.Printf("%s, %s \n", err.(awserr.Error).Code(), err.(awserr.Error).Error())
		return
	}

	// Pretty-print the response data.
	//fmt.Println(resp)
	for _, obj := range resp.Contents {
		if strings.HasPrefix(*obj.Key, "recommendations/output.txt/part-") {
			fmt.Println(GetObject(svc, *obj.Key))
		}
	}
}

func pull(s3serv *s3.S3, c config, p string) {
	if list, e := s3serv.ListObjects(&s3.ListObjectsInput{Bucket: &c.Bucket}); e == nil {
		for _, l := range list.Contents {
			log.Println(*l.Key)
			if obj, e := s3serv.GetObject(&s3.GetObjectInput{Bucket: &c.Bucket, Key: l.Key}); e == nil {
				os.MkdirAll(path.Dir(p+*l.Key), 0700)
				if f, e := os.OpenFile(p+*l.Key, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0666); e == nil {
					defer f.Close()
					defer obj.Body.Close()
					counter := &CountReader{input: obj.Body, co: c}
					counter.Init()
					io.Copy(f, counter)
					log.Println("written", *l.Key)
				} else {
					log.Println(e)
				}

			} else {
				log.Println(e)
			}
		}
	} else {
		log.Fatal(e)
	}

}

func S3ListObjectOutput(cli *s3.S3, bucket string) (listObjectOutput *s3.ListObjectsOutput, err error) {
	listObjectOutput, err = cli.ListObjects(&s3.ListObjectsInput{
		Bucket: aws.String(bucket),
	})

	return listObjectOutput, err
}

func emptyBucket(service *s3.S3, bucketName string) {
	objs, err := service.ListObjects(&s3.ListObjectsInput{Bucket: stringPtr(bucketName)})
	if err != nil {
		panic(err)
	}

	for _, o := range objs.Contents {
		_, err := service.DeleteObject(&s3.DeleteObjectInput{Bucket: stringPtr(bucketName), Key: o.Key})
		if err != nil {
			panic(err)
		}
	}
}

func listObjects(c *s3.S3, b, p string) ([]string, error) {
	l := make([]string, 0)
	resp, err := c.ListObjects(&s3.ListObjectsInput{
		Bucket: aws.String(b),
		Prefix: aws.String(p),
	})
	if err != nil {
		return l, err
	}
	for _, k := range resp.Contents {
		if strings.HasSuffix(*k.Key, fileSuffix) && *k.Size <= 5000 {
			l = append(l, *k.Key)
		}
	}
	return l, nil
}

func listObjects(client *s3.S3, bucket, prefix string, log *Log) (channel chan *s3.Object, err error) {
	request := &s3.ListObjectsInput{
		Bucket: &bucket,
		Prefix: &prefix,
	}

	channel = make(chan *s3.Object)

	if client == nil {
		close(channel)
		return
	}

	output, err := client.ListObjects(request)
	if err != nil {
		err = fmt.Errorf("S3 ListObjects: %s", err)
		close(channel)
		return
	}

	go func() {
		defer close(channel)

		for {
			for i := range output.Contents {
				object := output.Contents[i]
				channel <- object
				request.Marker = object.Key
			}

			if !*output.IsTruncated {
				break
			}

			if output, err = client.ListObjects(request); err != nil {
				log.Errorf("S3 ListObjects: %s", err)
				break
			}
		}
	}()

	return
}

func s3MassDelete(svc *s3.S3, bucket string, key string) {
	params := &s3.ListObjectsInput{
		Bucket: aws.String(bucket), // Required
		Prefix: aws.String(key),
	}
	resp, err := svc.ListObjects(params)

	if err != nil {
		fmt.Println(err.Error())
		return
	}

	s3keys := []*s3.ObjectIdentifier{}

	for _, m := range resp.Contents {
		//log.Printf("key: %s", *m.Key)
		s3keys = append(s3keys, &s3.ObjectIdentifier{Key: m.Key})
	}

	if len(s3keys) > 1000 {
		sliced := slicer(s3keys, 5)

		for _, keys := range sliced {
			vals := []*s3.ObjectIdentifier{}
			for _, v := range keys {
				vals = append(vals, v.(*s3.ObjectIdentifier))
			}
			result := s3Delete(svc, bucket, vals)

			if result == nil {
				log.Printf("s3 result is null")
			}
		}
	} else {
		result := s3Delete(svc, bucket, s3keys)

		if result == nil {
			log.Printf("s3 result is null")
		}
	}

}