Golang S3.GetObject方法代码示例

//get object from s3
func getObject(svc *s3.S3, bucket string, key string) string {
	params := &s3.GetObjectInput{
		Bucket: aws.String(bucket), // Required
		Key:    aws.String(key),    // Required
	}
	resp, err := svc.GetObject(params)

	if err != nil {
		// Print the error, cast err to awserr.Error to get the Code and
		// Message from an error.
		glog.Fatal(err.Error())
	}

	size := int(*resp.ContentLength)

	buffer := make([]byte, size)
	defer resp.Body.Close()
	var bbuffer bytes.Buffer
	for true {
		num, rerr := resp.Body.Read(buffer)
		if num > 0 {
			bbuffer.Write(buffer[:num])
		} else if rerr == io.EOF || rerr != nil {
			break
		}
	}
	return bbuffer.String()
}

func GetObject(svc *s3.S3, key string) string {
	//svc := s3.New(session.New(),&aws.Config{Region: aws.String("us-east-1")})

	params := &s3.GetObjectInput{
		Bucket: aws.String("ecomm-order-items"), // Required
		Key:    aws.String(key),                 // Required
	}
	resp, err := svc.GetObject(params)

	if err != nil {
		// Print the error, cast err to awserr.Error to get the Code and
		// Message from an error.
		glog.Fatal(err.Error())
	}

	// Pretty-print the response data.
	fmt.Print(resp)
	size := int(*resp.ContentLength)
	fmt.Println("size is ", size)
	buffer := make([]byte, size)
	defer resp.Body.Close()
	var bbuffer bytes.Buffer
	for true {

		num, rerr := resp.Body.Read(buffer)
		if num > 0 {
			//fmt.Println("times ",count)
			bbuffer.Write(buffer[:num])
			//bbuffer.WriteString(string(buffer[:num]))
		} else if rerr == io.EOF || rerr != nil {
			break
		}
	}
	return bbuffer.String()
}

func pull(s3serv *s3.S3, c config, p string) {
	if list, e := s3serv.ListObjects(&s3.ListObjectsInput{Bucket: &c.Bucket}); e == nil {
		for _, l := range list.Contents {
			log.Println(*l.Key)
			if obj, e := s3serv.GetObject(&s3.GetObjectInput{Bucket: &c.Bucket, Key: l.Key}); e == nil {
				os.MkdirAll(path.Dir(p+*l.Key), 0700)
				if f, e := os.OpenFile(p+*l.Key, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0666); e == nil {
					defer f.Close()
					defer obj.Body.Close()
					counter := &CountReader{input: obj.Body, co: c}
					counter.Init()
					io.Copy(f, counter)
					log.Println("written", *l.Key)
				} else {
					log.Println(e)
				}

			} else {
				log.Println(e)
			}
		}
	} else {
		log.Fatal(e)
	}

}

func (g *S3Getter) getObject(client *s3.S3, dst, bucket, key, version string) error {
	req := &s3.GetObjectInput{
		Bucket: aws.String(bucket),
		Key:    aws.String(key),
	}
	if version != "" {
		req.VersionId = aws.String(version)
	}

	resp, err := client.GetObject(req)
	if err != nil {
		return err
	}

	// Create all the parent directories
	if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil {
		return err
	}

	f, err := os.Create(dst)
	if err != nil {
		return err
	}
	defer f.Close()

	_, err = io.Copy(f, resp.Body)
	return err
}

// GetS3Object returns the object output for the given object key
func GetS3Object(client *s3.S3, bucket, objKey string) (*s3.GetObjectOutput, error) {
	logger.Debug("GetS3Object: ", bucket, "-", objKey)
	params := &s3.GetObjectInput{
		Bucket: aws.String(bucket), // Required
		Key:    aws.String(objKey), // Required
		// IfMatch:                    aws.String("IfMatch"),
		// IfModifiedSince:            aws.Time(time.Now()),
		// IfNoneMatch:                aws.String("IfNoneMatch"),
		// IfUnmodifiedSince:          aws.Time(time.Now()),
		// Range:                      aws.String("Range"),
		// RequestPayer:               aws.String("RequestPayer"),
		// ResponseCacheControl:       aws.String("ResponseCacheControl"),
		// ResponseContentDisposition: aws.String("ResponseContentDisposition"),
		// ResponseContentEncoding:    aws.String("ResponseContentEncoding"),
		// ResponseContentLanguage:    aws.String("ResponseContentLanguage"),
		// ResponseContentType:        aws.String("ResponseContentType"),
		// ResponseExpires:            aws.Time(time.Now()),
		// SSECustomerAlgorithm:       aws.String("SSECustomerAlgorithm"),
		// SSECustomerKey:             aws.String("SSECustomerKey"),
		// SSECustomerKeyMD5:          aws.String("SSECustomerKeyMD5"),
		// VersionId:                  aws.String("ObjectVersionId"),
	}

	return client.GetObject(params)
}

func getObject(client *s3.S3, bucket string, key *string) (output *s3.GetObjectOutput, err error) {
	request := &s3.GetObjectInput{
		Bucket: &bucket,
		Key:    key,
	}

	return client.GetObject(request)
}

func get_object(svc *s3.S3, bucket *string, key *string) (io.ReadCloser, error) {
    params := &s3.GetObjectInput{
        Bucket: bucket,
        Key:    key,
    }
    resp, err := svc.GetObject(params)
    if err != nil { return nil, err }
    return resp.Body, nil
}

func getBlob(c *s3.S3, b, k string) ([]byte, error) {
	resp, err := c.GetObject(&s3.GetObjectInput{
		Bucket: aws.String(b),
		Key:    aws.String(k),
	})
	if err != nil {
		return nil, err
	}
	blob := make([]byte, *resp.ContentLength)
	_, err = resp.Body.Read(blob)
	return blob, nil
}

func get(service *s3.S3, bucketName string, key, path string) {
	obj, err := service.GetObject(&s3.GetObjectInput{
		Bucket: stringPtr(bucketName),
		Key:    stringPtr(key),
	})
	if err != nil {
		panic(err)
	}
	defer obj.Body.Close()
	f, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0755)
	if err != nil {
		panic(err)
	}
	defer f.Close()
	_, err = io.Copy(f, obj.Body)
	if err != nil {
		panic(err)
	}
}

// Tries to download the file from S3. This uses GetObject which
// uses a single HTTP stream, rather than an s3Manager.Downloader,
// which uses multiple streams. We generally have to calculate
// both an md5 and a sha256 checksum on download, and we're choosing
// to write the file and do the checksums all in one pass. The
// s3Manager.Downloader's multiple concurrent connections produce
// faster downloads, but requires a WrterAt interface, which the
// hashing algorithms don't provide. When we're working with
// multi-gigabyte files, we really don't want to have to read them
// again to produce the checksums.
func (client *S3Download) tryDownload(service *s3.S3, params *s3.GetObjectInput) error {
	resp, err := service.GetObject(params)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	client.Response = resp

	// Create the download directory and open a file for writing.
	writers := make([]io.Writer, 0)
	if client.LocalPath == os.DevNull {
		writers = append(writers, ioutil.Discard)
	} else {
		err = os.MkdirAll(filepath.Dir(client.LocalPath), 0755)
		if err != nil {
			return err
		}
		outputFile, err := os.Create(client.LocalPath)
		if err != nil {
			return err
		}
		writers = append(writers, outputFile)
		defer outputFile.Close()
	}

	// Create a writer to write the contents to the file,
	// and optionally to pass the bitstream through the
	// md5 and sha256 algorithms while we're at it.
	var multiWriter io.Writer
	var md5Hash hash.Hash
	var sha256Hash hash.Hash
	if client.CalculateMd5 {
		md5Hash = md5.New()
		writers = append(writers, md5Hash)
	}
	if client.CalculateSha256 {
		sha256Hash = sha256.New()
		writers = append(writers, sha256Hash)
	}
	multiWriter = io.MultiWriter(writers...)

	// Copy the file, with several tries. On larger files,
	// we often get a "connection reset by peer" error.
	// Better to retry a few times now than throw this
	// back into the work queue.
	for attemptNumber := 0; attemptNumber < 5; attemptNumber++ {
		client.BytesCopied, err = io.Copy(multiWriter, resp.Body)
		if err == nil {
			break
		}
	}
	if err != nil {
		return err
	}

	// Set the checksums, if needed...
	if client.CalculateMd5 {
		client.Md5Digest = fmt.Sprintf("%x", md5Hash.Sum(nil))
	}
	if client.CalculateSha256 {
		client.Sha256Digest = fmt.Sprintf("%x", sha256Hash.Sum(nil))
	}

	// No errors.
	return nil
}