Golang s3.Bucket类代码示例

func putFiles(bucket *s3.Bucket, c <-chan *Item) {
	var wg sync.WaitGroup
	wg.Add(options.Concurrency)
	for i := 0; i < options.Concurrency; i++ {
		go func() {
			for item := range c {
				func() {
					f, err := os.Open(item.Path)
					if err != nil {
						log.Printf("Pushing %s failed: %s", item.Path, err)
					}
					defer f.Close()

					path := item.Path[len(item.Prefix)+1:]
					err = bucket.PutReader(options.Put.Prefix+path, f, item.FileInfo.Size(), mime.TypeByExtension(filepath.Ext(item.Path)), s3.BucketOwnerFull)
					if err != nil {
						log.Printf("Uploading %s failed: %s", path, err)
						return
					}
					log.Printf("Uploading %s done", path)
				}()
			}
			wg.Done()
		}()
	}
	wg.Wait()
}

func generateThumbnails(file multipart.File, userId, pathPrefix string, bucket *s3.Bucket) error {
	file.Seek(0, 0)

	img, _, err := image.Decode(file)
	if err != nil {
		log.Println("Error decoding image", err)
		return err
	}

	var buf bytes.Buffer
	for _, i := range THUMBNAIL_SIZES {
		resized := resize.Resize(uint(i), 0, img, resize.Lanczos3)
		err = jpeg.Encode(&buf, resized, nil)
		if err != nil {
			return err
		}
		path := fmt.Sprintf("%s/%d.jpg", pathPrefix, i)
		err = bucket.Put(path, buf.Bytes(), "image/jpeg", s3.PublicRead)
		if err != nil {
			return err
		}
		buf.Reset()
	}

	return err
}

func hashRemote(bucket *s3.Bucket, path string) []byte {
	file, err := bucket.GetReader(path)
	if err != nil {
		return nil
	}
	return hashFile(file)
}

func listBucketFiles(bucket *s3.Bucket) <-chan *Item {
	c := make(chan *Item)
	go func() {
		marker := ""
		for {
			resp, err := bucket.List(options.Get.Prefix, "", marker, 1000)
			if err != nil {
				log.Printf("Could not list items in bucket: %s", err)
			}
			for _, item := range resp.Contents {
				c <- &Item{
					Prefix:   options.Get.Prefix,
					Path:     item.Key,
					FileInfo: nil,
				}
				marker = item.Key
			}
			if !resp.IsTruncated {
				break
			}
		}
		close(c)
	}()
	return c
}

func Get(file string, bucket *s3.Bucket, path string) {
	data, err := bucket.Get(path)
	if err != nil {
		panic(err.Error())
	}
	perms := os.FileMode(0644)

	err = ioutil.WriteFile(file, data, perms)
	if err != nil {
		panic(err.Error())
	}
}

func upload(directory string, f os.FileInfo, bucket *s3.Bucket, permission s3.ACL) {
	log.Println("uploading " + f.Name())
	data, err := ioutil.ReadFile(path.Join(directory, f.Name()))
	if err != nil {
		panic(err.Error())
	}
	err = bucket.Put(f.Name(), data, "", permission)
	if err != nil {
		panic(err.Error())
	}
	log.Println("finished uploading " + f.Name())
}

func upload(bucket *s3.Bucket, checksum, filename string) {
	f, err := os.Open(filename)
	handleErr(err)
	defer f.Close()
	path := fmt.Sprintf("%s/%s", checksum, filename)
	fi, err := f.Stat()
	handleErr(err)
	err = bucket.PutReader(path, f, fi.Size(), "binary/octet-stream", s3.PublicRead)
	if err != nil {
		log.Fatalf("Go makes me sad because of: %s", err)
	}
	log.Printf("Uploaded %s", path)
}

func Put(bucket *s3.Bucket, path string, file string) {
	contType := "binary/octet-stream"
	Perms := s3.ACL("private")

	data, err := ioutil.ReadFile(file)
	if err != nil {
		panic(err.Error())
	}

	err = bucket.Put(path, data, contType, Perms)
	if err != nil {
		panic(err.Error())
	}
}

func killBucket(b *s3.Bucket) {
	var err error
	for attempt := attempts.Start(); attempt.Next(); {
		err = b.DelBucket()
		if err == nil {
			return
		}
		if _, ok := err.(*net.DNSError); ok {
			return
		}
		e, ok := err.(*s3.Error)
		if ok && e.Code == "NoSuchBucket" {
			return
		}
		if ok && e.Code == "BucketNotEmpty" {
			// Errors are ignored here. Just retry.
			resp, err := b.List("", "", "", 1000)
			if err == nil {
				for _, key := range resp.Contents {
					_ = b.Del(key.Key)
				}
			}
			multis, _, _ := b.ListMulti("", "")
			for _, m := range multis {
				_ = m.Abort()
			}
		}
	}
	message := "cannot delete test bucket"
	if err != nil {
		message += ": " + err.Error()
	}
	panic(message)
}

func GetReader(b *s3.Bucket, path string) (reader *TweetReader, err error) {
	body, err := b.GetReader(path)
	if err != nil {
		return nil, err
	}

	gr, err := gzip.NewReader(body)
	if err != nil {
		return nil, err
	}

	reader = NewTweetReaderSerial(bufio.NewReaderSize(gr, 16*4096))

	return reader, err
}

// Sends this upload to the given S3 bucket
func (u *Upload) send(b *s3.Bucket) error {
	f, err := os.Open(u.LocalFile)
	if err != nil {
		return err
	}
	defer f.Close()

	info, err := os.Stat(u.LocalFile)
	if err != nil {
		return err
	}

	r := bufio.NewReader(f)
	err = b.PutReader(u.RemoteFile, r, info.Size(), u.GetContentType(), u.Access)
	if err != nil {
		return err
	}

	return nil
}

func getFiles(bucket *s3.Bucket, c <-chan *Item) {
	var wg sync.WaitGroup
	wg.Add(options.Concurrency)
	for i := 0; i < options.Concurrency; i++ {
		go func() {
			for item := range c {
				func() {
					itempath := item.Path[len(item.Prefix):]
					dirname, fname := filepath.Split(itempath)
					dirname = filepath.Join(options.Remainder[0], dirname)

					err := os.MkdirAll(dirname, os.FileMode(0755))
					if err != nil {
						log.Printf("Could not create target folder %s: %s", dirname, err)
						return
					}

					f, err := os.Create(filepath.Join(dirname, fname))
					if err != nil {
						log.Printf("Opening %s failed: %s", item.Path, err)
						return
					}
					defer f.Close()

					rc, err := bucket.GetReader(item.Path)
					if err != nil {
						log.Printf("Downloading %s failed: %s", item.Path, err)
						return
					}
					defer rc.Close()
					io.Copy(f, rc)
					log.Printf("Downloading %s done", item.Path)
				}()
			}
			wg.Done()
		}()
	}
	wg.Wait()
}

func syncPath(acl, localPath, path string, bucket *s3.Bucket) {
	s3Path := strings.Replace(path, localPath, "", -1)
	file, err := os.Open(path)
	if err != nil {
		fmt.Println("Error opening local file, Failed to sync '" + path + "'")
		return
	}
	info, err := file.Stat()
	if err != nil {
		fmt.Println("Error stating local file, Failed to sync '" + path + "'")
		return
	}
	length := info.Size()
	pathSplit := strings.Split(path, ".")
	ext := pathSplit[len(pathSplit)-1]
	mtype := mime.TypeByExtension(ext)
	puterr := bucket.PutReader(s3Path, file, length, mtype, s3.ACL(acl))
	if puterr != nil {
		fmt.Println("Failed to sync: " + s3Path)
		return
	}
	fmt.Println("Synced: " + s3Path)

}