Golang s3manager.NewDownloader函数代码示例

func NewS3(awsId, awsSecret, s3Bucket, s3Region string) Source {
	cred := credentials.NewStaticCredentials(awsId, awsSecret, "")
	session := session.New(&aws.Config{Region: aws.String(s3Region), Credentials: cred})

	s3manager.NewDownloader(session)

	return &S3{s3manager.NewDownloader(session), s3Bucket}
}

func downloadFromS3(s3path string, localPath string, scanned func(string, bool)) error {
	outputPath := scratchPath(localPath)

	sess := session.New(&aws.Config{Region: aws.String("us-west-2")})
	downloader := s3manager.NewDownloader(sess, func(d *s3manager.Downloader) {
		d.PartSize = 5 * 1024 * 1024 // 5MB per part
	})

	file, err := os.Create(outputPath)
	if err != nil {
		return err
	}

	log.Println("Starting download")

	svc := s3.New(sess)
	resp, err := svc.HeadObject(&s3.HeadObjectInput{
		Bucket: aws.String("smick-media-output"),
		Key:    aws.String(s3path),
	})
	if err != nil {
		log.Println("Error getting head:", err)
	}
	log.Println(resp)

	downloader.Download(file, &s3.GetObjectInput{
		Bucket: aws.String("smick-media-output"),
		Key:    aws.String(s3path),
	}, func(d *s3manager.Downloader) {
		log.Println("yeye")
	})
	log.Println("Downloaded")
	return nil
}

func GetAction(c *cli.Context) {
	if len(c.Args()) != 2 {
		log.Fatal("get s3path localpath")
	}

	s3path, err := NewS3Path(c.Args().Get(0))
	if err != nil {
		log.Fatal(err)
	}

	localpath := c.Args().Get(1)
	if localpath == "" {
		log.Fatal("get s3path localpath")
	}

	log.Printf("s3path Bucket:%v Prefix:%v", s3path.Bucket, s3path.Prefix)
	parallel := c.Int("parallel")

	manager := s3manager.NewDownloader(nil)
	d := NewDownloader(s3path, localpath, parallel, manager)

	client := s3.New(nil)
	params := &s3.ListObjectsInput{Bucket: &s3path.Bucket, Prefix: &s3path.Prefix}
	err = client.ListObjectsPages(params, d.eachPage)
	if err != nil {
		log.Fatal(err)
	}
	d.Wait()
}

// Get downloads the specified release from S3 bucket, as described in
// the passed in config, and returns the location of the downloaded
// release and an error, if any
func Get(conf *S3Config) (string, error) {
	defaults.DefaultConfig.Region = aws.String(*conf.Region)

	tmpDir := os.TempDir()

	destFile := filepath.Join(tmpDir, *conf.Revision)

	downloadFile, err := os.Create(destFile)
	if err != nil {
		return "", err
	}

	filePath := filepath.Join(*conf.RevisionPath, *conf.Revision)
	downloader := s3manager.NewDownloader(nil)
	_, err = downloader.Download(
		downloadFile,
		&s3.GetObjectInput{
			Bucket: conf.Bucket,
			Key:    &filePath,
		})
	if err != nil {
		return "", err
	}

	return destFile, nil
}

// getRemoteBackup is used to pull backups from S3
func getRemoteBackup(r *Restore, conf *config.Config) {
	s3Conn := session.New(&aws.Config{Region: aws.String(string(conf.S3Region))})

	r.LocalFilePath = fmt.Sprintf("%v/%v", conf.TmpDir, r.RestorePath)

	localFileDir := filepath.Dir(r.LocalFilePath)

	err := os.MkdirAll(localFileDir, 0755)
	if err != nil {
		log.Fatalf("[ERR] Unable to create local restore directory!: %v", err)
	}

	outFile, err := os.Create(r.LocalFilePath)
	if err != nil {
		log.Fatalf("[ERR] Unable to create local restore temp file!: %v", err)
	}

	// Create the params to pass into the actual downloader
	params := &s3.GetObjectInput{
		Bucket: &conf.S3Bucket,
		Key:    &r.RestorePath,
	}

	log.Printf("[INFO] Downloading %v%v from S3 in %v", string(conf.S3Bucket), r.RestorePath, string(conf.S3Region))
	downloader := s3manager.NewDownloader(s3Conn)
	_, err = downloader.Download(outFile, params)
	if err != nil {
		log.Fatalf("[ERR] Could not download file from S3!: %v", err)
	}
	outFile.Close()
	log.Print("[INFO] Download completed")
}

func (D *Data) GetFile(src string, dst string) error {
	log.WithFields(
		log.Fields{"src": src,
			"dst":     dst,
			"S3 Data": D,
		}).Info("S3: GetFile")

	file, err := os.Create(dst)
	if err != nil {
		log.WithFields(log.Fields{"dst": dst}).Error("S3: Cannot create file")
		return err
	}
	defer file.Close()

	s3Param := &s3.GetObjectInput{
		Bucket: aws.String(D.BktName),
		Key:    aws.String(src)}
	downloader := s3manager.NewDownloader(D.s3Sess)
	n, err := downloader.Download(file, s3Param)
	if err != nil {
		log.WithFields(
			log.Fields{"S3": D, "Key": src, "Error": err}).Error("S3: Cannot download")
		os.Remove(dst)
		return err
	}

	log.WithFields(
		log.Fields{"S3": D, "Src": src, "Dst": dst, "Bytes recvd": n}).Debug("S3: GetFile")
	return nil
}

func (client *s3client) DownloadFile(
	bucketName string,
	remotePath string,
	localPath string,
) error {
	downloader := s3manager.NewDownloader(client.session)

	localFile, err := os.Create(localPath)
	if err != nil {
		return err
	}
	defer localFile.Close()

	getObject := &s3.GetObjectInput{
		Bucket: aws.String(bucketName),
		Key:    aws.String(remotePath),
	}

	_, err = downloader.Download(localFile, getObject)
	if err != nil {
		return err
	}

	return nil
}

func (client *s3client) DownloadFile(bucketName string, remotePath string, versionID string, localPath string) error {
	downloader := s3manager.NewDownloader(&s3manager.DownloadOptions{
		S3: client.client,
	})

	localFile, err := os.Create(localPath)
	if err != nil {
		return err
	}
	defer localFile.Close()

	getObject := &s3.GetObjectInput{
		Bucket: aws.String(bucketName),
		Key:    aws.String(remotePath),
	}

	if versionID != "" {
		getObject.VersionId = aws.String(versionID)
	}

	_, err = downloader.Download(localFile, getObject)
	if err != nil {
		return err
	}

	return nil
}

func TestDownloadError(t *testing.T) {
	s, names, _ := dlLoggingSvc([]byte{1, 2, 3})
	opts := &s3manager.DownloadOptions{S3: s, PartSize: 1, Concurrency: 1}

	num := 0
	s.Handlers.Send.PushBack(func(r *request.Request) {
		num++
		if num > 1 {
			r.HTTPResponse.StatusCode = 400
			r.HTTPResponse.Body = ioutil.NopCloser(bytes.NewReader([]byte{}))
		}
	})

	d := s3manager.NewDownloader(opts)
	w := &aws.WriteAtBuffer{}
	n, err := d.Download(w, &s3.GetObjectInput{
		Bucket: aws.String("bucket"),
		Key:    aws.String("key"),
	})

	assert.NotNil(t, err)
	assert.Equal(t, int64(1), n)
	assert.Equal(t, []string{"GetObject", "GetObject"}, *names)
	assert.Equal(t, []byte{1}, w.Bytes())
}

func validate(t *testing.T, key string, md5value string) {
	mgr := s3manager.NewDownloader(nil)
	params := &s3.GetObjectInput{Bucket: bucketName, Key: &key}

	w := newDLWriter(1024 * 1024 * 20)
	n, err := mgr.Download(w, params)
	assert.NoError(t, err)
	assert.Equal(t, md5value, fmt.Sprintf("%x", md5.Sum(w.buf[0:n])))
}

func main() {
	jobs := make(chan *s3.Object)

	s3Url := os.Args[1]
	destDir := os.Args[2]

	u, err := url.Parse(s3Url)
	if err != nil {
		log.Fatal(err)
	}

	s3Bucket := u.Host
	s3Prefix := u.Path[1:]

	err = os.MkdirAll(destDir, 0700)
	if err != nil {
		log.Fatal(err)
	}

	session := session.New(&aws.Config{Region: aws.String(awsRegion)})
	svc := s3.New(session)
	downloader := s3manager.NewDownloader(session)

	params := &s3.ListObjectsInput{
		Bucket: aws.String(s3Bucket),
		Prefix: aws.String(s3Prefix),
	}

	var wg sync.WaitGroup
	for w := 1; w <= concurrency; w++ {
		wg.Add(1)
		go func(w int) {
			worker(w, jobs, downloader, s3Bucket, destDir)
			defer wg.Done()
		}(w)
	}

	log.Printf("Looking for objects in bucket: %s, prefix: %s", s3Bucket, s3Prefix)

	err = svc.ListObjectsPages(params, func(page *s3.ListObjectsOutput, lastPage bool) bool {
		for _, object := range page.Contents {
			jobs <- object
		}
		objectsCount += len(page.Contents)
		return true
	})

	close(jobs)
	wg.Wait()

	if err != nil {
		log.Fatal(err)
	}

	fmt.Printf("Found %d objects to download.\n", objectsCount)
}

func NewS3Connection(connection *s3.S3, cacheDirectory string, s3BucketName string,
	options *s3manager.DownloadOptions) S3Connection {
	conn := S3Connection{
		Connection:     connection,
		BucketName:     s3BucketName,
		CacheDirectory: cacheDirectory,
	}
	conn.Downloader = s3manager.NewDownloader(options)
	return conn
}

// Download using to download file from S3 by bucket and key
func Download(session *session.Session, bucket, key, path string) {
	file, err := os.Create(path)
	logger.Process(err, "Can't create file")
	defer file.Close()
	client := s3manager.NewDownloader(session)
	_, err = client.Download(file, &s3.GetObjectInput{
		Bucket: aws.String(bucket),
		Key:    aws.String(key),
	})
	logger.Process(err, "Can't retrieve object from S3")
}

/*
Download downloads a file from S3.

This is merely a wrapper around the aws-sdk-go downloader. It allows us to
isolate the aws-sdk-go dependencies and unify error handling.
*/
func Download(file *os.File, bucket, key string) error {
	downloader := s3manager.NewDownloader(session.New(&aws.Config{Region: aws.String("us-east-1")}))
	numBytes, err := downloader.Download(file,
		&s3.GetObjectInput{
			Bucket: aws.String(bucket),
			Key:    aws.String(key),
		})
	if err != nil {
		return err
	}
	log.Println("Downloaded", numBytes, "bytes")
	return nil
}

func TestDownloadZero(t *testing.T) {
	s, names, ranges := dlLoggingSvc([]byte{})

	opts := &s3manager.DownloadOptions{S3: s}
	d := s3manager.NewDownloader(opts)
	w := &aws.WriteAtBuffer{}
	n, err := d.Download(w, &s3.GetObjectInput{
		Bucket: aws.String("bucket"),
		Key:    aws.String("key"),
	})

	assert.Nil(t, err)
	assert.Equal(t, int64(0), n)
	assert.Equal(t, []string{"GetObject"}, *names)
	assert.Equal(t, []string{"bytes=0-5242879"}, *ranges)
}