Golang s3manager.NewUploader函数代码示例

func main() {
	flag.Parse()
	if *filepath == "" || *bucket == "" {
		fmt.Println("Please specify correct parameters!")
		fmt.Println("You should specify:")
		fmt.Println("-path with path to file you want to upload")
		fmt.Println("-bucket name of bucket in S3 where you want to upload")
		os.Exit(1)
	}

	file, err := os.Open(*filepath)
	if err != nil {
		fmt.Println("Failed to open a file.", err)
		os.Exit(1)
	}

	session := session.New(createConfig())
	service := s3manager.NewUploader(session)

	resp, err := service.Upload(&s3manager.UploadInput{
		Bucket: aws.String(*bucket),
		Key:    aws.String("/" + getFileName(*filepath)),
		Body:   file,
	})
	if err != nil {
		fmt.Println("Failed to upload a file.", err)
		os.Exit(1)
	}

	fmt.Println("---------------------")
	fmt.Println("File was successfully uploaded!")
	fmt.Println("Location:", resp.Location)
}

func (this *StorageUtil) upload(fileName string, fileLocation string, fingerPrint string) (string, error) {
	f, err := os.Open(fileLocation)
	defer f.Close()

	if err != nil {
		return "", err
	}

	metadata := map[string]*string{}
	metadata["fingerprint"] = &fingerPrint

	uploader := s3manager.NewUploader(this.ssn)

	result, err := uploader.Upload(&s3manager.UploadInput{
		Bucket:   &this.Bucket,
		Key:      &fileName,
		Body:     f,
		Metadata: metadata,
	})

	if err != nil {
		return "", err
	} else {
		return result.Location, nil
	}
}

func main() {
	flag.Parse()
	if *filesPath == "" || *bucket == "" {
		fmt.Println("Please specify correct parameters!")
		fmt.Println("You should specify:")
		fmt.Println("-path with path to file you want to upload")
		fmt.Println("-bucket name of bucket in S3 where you want to upload")
		os.Exit(1)
	}

	file, err := os.Open(*filesPath)
	if err != nil {
		log.Fatal("Failed to open a file with an error: ", err)
	}
	defer file.Close()
	info, err := file.Stat()
	if err != nil {
		log.Fatal("Failed to get info about file with an error: ", err)
	}

	session := session.New(createConfig())
	service := s3manager.NewUploader(session)

	switch mode := info.Mode(); {
	case mode.IsDir():
		uploadDirectory(*service, *file)
	case mode.IsRegular():
		uploadFile(*service, *uploadPath+getFileName(*filesPath), file)
	}
}

// Update the Object from in with modTime and size
func (o *Object) Update(in io.Reader, modTime time.Time, size int64) error {
	uploader := s3manager.NewUploader(o.fs.ses, func(u *s3manager.Uploader) {
		u.Concurrency = 2
		u.LeavePartsOnError = false
		u.S3 = o.fs.c
	})

	// Set the mtime in the meta data
	metadata := map[string]*string{
		metaMtime: aws.String(swift.TimeToFloatString(modTime)),
	}

	// Guess the content type
	contentType := fs.MimeType(o)

	key := o.fs.root + o.remote
	req := s3manager.UploadInput{
		Bucket:      &o.fs.bucket,
		ACL:         &o.fs.perm,
		Key:         &key,
		Body:        in,
		ContentType: &contentType,
		Metadata:    metadata,
		//ContentLength: &size,
	}
	_, err := uploader.Upload(&req)
	if err != nil {
		return err
	}

	// Read the metadata from the newly created object
	o.meta = nil // wipe old metadata
	err = o.readMetaData()
	return err
}

// Upload upload object
func (service Service) Upload(key string, filePath string) error {
	if key == "" {
		return fmt.Errorf("key must be a nonempty string")
	}
	if filePath == "" {
		return fmt.Errorf("filePath must be a nonempty string")
	}

	file, err := os.Open(filePath)
	if err != nil {
		return err
	}

	defer file.Close()
	uploader := s3manager.NewUploader(session.New())
	_, err = uploader.Upload(&s3manager.UploadInput{
		Bucket:      aws.String(service.bucketSource),
		Key:         aws.String(key),
		ACL:         aws.String(s3.ObjectCannedACLPublicRead),
		ContentType: aws.String(mime.TypeByExtension(path.Ext(key))),
		Body:        file,
	})

	return err
}

func (client *s3client) UploadFile(bucketName string, remotePath string, localPath string) (string, error) {
	uploader := s3manager.NewUploader(client.session)

	stat, err := os.Stat(localPath)
	if err != nil {
		return "", err
	}

	localFile, err := os.Open(localPath)
	if err != nil {
		return "", err
	}

	defer localFile.Close()

	progress := client.newProgressBar(stat.Size())

	progress.Start()
	defer progress.Finish()

	uploadOutput, err := uploader.Upload(&s3manager.UploadInput{
		Bucket: aws.String(bucketName),
		Key:    aws.String(remotePath),
		Body:   progress.NewProxyReader(localFile),
	})
	if err != nil {
		return "", err
	}

	if uploadOutput.VersionID != nil {
		return *uploadOutput.VersionID, nil
	}

	return "", nil
}

func (client *s3client) UploadFile(bucketName string, remotePath string, localPath string) (string, error) {
	uploader := s3manager.NewUploader(&s3manager.UploadOptions{
		S3: client.client,
	})

	localFile, err := os.Open(localPath)
	if err != nil {
		return "", err
	}

	defer localFile.Close()

	// uploadOutput, err := uploader.Upload(&s3manager.UploadInput{
	_, err = uploader.Upload(&s3manager.UploadInput{
		Bucket: aws.String(bucketName),
		Key:    aws.String(remotePath),
		Body:   localFile,
	})
	if err != nil {
		return "", err
	}

	// if uploadOutput.VersionID != nil {
	// return *uploadOutput.VersionID, nil
	// }

	return "", nil
}

// Write the local backup file to S3.
// There are no tests for this remote operation
func (b *Backup) writeBackupRemote() {
	s3Conn := session.New(&aws.Config{Region: aws.String(string(b.Config.S3Region))})

	t := time.Unix(b.StartTime, 0)

	b.RemoteFilePath = fmt.Sprintf("%s/%v/%d/%v/%v", b.Config.ObjectPrefix, t.Year(), t.Month(), t.Day(), filepath.Base(b.FullFilename))

	// re-read the compressed file.  There is probably a better way to do this
	localFileContents, err := ioutil.ReadFile(b.FullFilename)
	if err != nil {
		log.Fatalf("[ERR] Could not read compressed file!: %v", err)
	}

	// Create the params to pass into the actual uploader
	params := &s3manager.UploadInput{
		Bucket: &b.Config.S3Bucket,
		Key:    &b.RemoteFilePath,
		Body:   bytes.NewReader(localFileContents),
	}

	if b.Config.S3ServerSideEncryption != "" {
		params.ServerSideEncryption = &b.Config.S3ServerSideEncryption
	}

	if b.Config.S3KmsKeyID != "" {
		params.SSEKMSKeyId = &b.Config.S3KmsKeyID
	}

	log.Printf("[INFO] Uploading %v/%v to S3 in %v", string(b.Config.S3Bucket), b.RemoteFilePath, string(b.Config.S3Region))
	uploader := s3manager.NewUploader(s3Conn)
	_, err = uploader.Upload(params)
	if err != nil {
		log.Fatalf("[ERR] Could not upload to S3!: %v", err)
	}
}

func uploadPackage(config *Config) error {
	key := "package.zip"
	uploadfile, err := os.Open(key)
	if err != nil {
		return err
	}

	//rlu := &rateLimitUploader{ fh: uploadfile }

	fmt.Println("Uploading package.zip to", config.S3Bucket, "bucket...")
	uploader := s3manager.NewUploader(session.New())
	_, err = uploader.Upload(&s3manager.UploadInput{
		Bucket: &config.S3Bucket,
		Key:    &key,
		//Body:   rlu,
		Body: uploadfile,
	})

	if err != nil {
		panic(err)
	}

	fmt.Println("Uploaded package.zip to", config.S3Bucket, "bucket.")

	return nil
}

func Upload(conf *Config, file io.Reader, fileName string, isPNG bool) (string, error) {
	s3Config := aws.NewConfig().
		WithCredentials(credentials.NewStaticCredentials(conf.AccessKeyID, conf.SecretAccessKey, ""))

	forcePathStyle := true
	s3Config.S3ForcePathStyle = &forcePathStyle

	if conf.AwsRegion == "" {
		s3Config = s3Config.WithRegion(" ").WithEndpoint(conf.Endpoint)
	} else {
		s3Config = s3Config.WithRegion(conf.AwsRegion)
	}

	sess := session.New(s3Config)

	uploader := s3manager.NewUploader(sess)

	upParams := &s3manager.UploadInput{
		ACL:    aws.String("public-read"),
		Bucket: &conf.BucketName,
		Key:    &fileName,
		Body:   file,
	}

	if isPNG {
		upParams.ContentType = aws.String("image/png")
	}

	result, err := uploader.Upload(upParams)
	if err != nil {
		return "", fmt.Errorf("Failed to upload file, err: %s", err.Error())
	}

	return result.Location, nil
}

func TestUploadOrderMultiFailureLeaveParts(t *testing.T) {
	s, ops, _ := loggingSvc(emptyList)
	s.Handlers.Send.PushBack(func(r *service.Request) {
		switch data := r.Data.(type) {
		case *s3.UploadPartOutput:
			if *data.ETag == "ETAG2" {
				r.HTTPResponse.StatusCode = 400
			}
		}
	})

	mgr := s3manager.NewUploader(&s3manager.UploadOptions{
		S3:                s,
		Concurrency:       1,
		LeavePartsOnError: true,
	})
	_, err := mgr.Upload(&s3manager.UploadInput{
		Bucket: aws.String("Bucket"),
		Key:    aws.String("Key"),
		Body:   bytes.NewReader(make([]byte, 1024*1024*12)),
	})

	assert.Error(t, err)
	assert.Equal(t, []string{"CreateMultipartUpload", "UploadPart", "UploadPart"}, *ops)
}

func fileToBucket(filename, bucket string) (size int64, err error) {
	defer Track("fileToBucket", Now(), debugOut)

	// Open the file
	file, err := os.Open(filename)
	if err != nil {
		return
	}
	defer file.Close()

	// Get the filesize
	fi, ferr := file.Stat()
	if ferr == nil {
		size = fi.Size()
	}

	// Extract the basename
	baseFilename := filepath.Base(filename)

	// Setup the uploader, and git'r'done
	svc := s3manager.NewUploader(AWSSession)
	_, err = svc.Upload(&s3manager.UploadInput{
		Bucket: aws.String(bucket),
		Key:    aws.String(baseFilename),
		Body:   file,
	})

	return
}

func TestUploadFailCleanup(t *testing.T) {
	svc := s3.New(nil)

	// Break checksum on 2nd part so it fails
	part := 0
	svc.Handlers.Build.PushBack(func(r *request.Request) {
		if r.Operation.Name == "UploadPart" {
			if part == 1 {
				r.HTTPRequest.Header.Set("X-Amz-Content-Sha256", "000")
			}
			part++
		}
	})

	key := "12mb-leave"
	mgr := s3manager.NewUploader(&s3manager.UploadOptions{
		S3:                svc,
		LeavePartsOnError: false,
	})
	_, err := mgr.Upload(&s3manager.UploadInput{
		Bucket: bucketName,
		Key:    &key,
		Body:   bytes.NewReader(integBuf12MB),
	})
	assert.Error(t, err)
	uploadID := ""
	if merr, ok := err.(s3manager.MultiUploadFailure); ok {
		uploadID = merr.UploadID()
	}
	assert.NotEmpty(t, uploadID)

	_, err = svc.ListParts(&s3.ListPartsInput{
		Bucket: bucketName, Key: &key, UploadId: &uploadID})
	assert.Error(t, err)
}

func handleMultipartForm(req *http.Request, folderName string) (err error) {
	// 10MB allocated for files
	const _10mb = (1 << 22) * 24
	if err = req.ParseMultipartForm(_10); err != nil {
		return
	}

	uploader := s3manager.NewUploader(sess)
	for _, fileHeaders := range req.MultipartForm.File {
		for _, header := range fileHeaders {
			var file multipart.File
			if file, err = header.Open(); err != nil {
				return
			}

			_, err = uploader.Upload(&s3manager.UploadInput{
				Bucket: aws.String(conf.S3.BucketName),
				Key:    aws.String(fmt.Sprintf("%s/%s", folderName, header.Filename)),
				Body:   file,
			})
			if err != nil {
				return
			}
		}
	}
	return
}

// Save will upload a file to S3
func (a *Amazon) Save(filepath, filename, mime string, expire bool) (err error) {

	file, err := os.Open(filepath)
	if err != nil {
		return errors.New("problem opening file for s3")
	}
	defer file.Close()

	uploader := s3manager.NewUploader(a.session)

	// default cachecontrol header
	var cache = "public, max-age=31536000"

	// if we want the file to not be cached
	if expire {
		cache = "public, max-age=0"
	}

	params := &s3manager.UploadInput{
		Bucket:               aws.String(config.Settings.Amazon.Bucket),
		Key:                  aws.String(filename),
		Body:                 file,
		ContentType:          aws.String(mime),
		CacheControl:         aws.String(cache),
		ServerSideEncryption: aws.String(s3.ServerSideEncryptionAes256),
	}

	_, err = uploader.Upload(params)

	return

}