Scala AmazonS3类代码示例

//设置package包名称以及导入依赖的类
package utils

import com.amazonaws.auth.AWSCredentialsProvider
import com.amazonaws.services.autoscaling.{AmazonAutoScaling, AmazonAutoScalingClient}
import com.amazonaws.services.cloudformation.{AmazonCloudFormation, AmazonCloudFormationClient}
import com.amazonaws.services.elasticloadbalancing.{AmazonElasticLoadBalancing, AmazonElasticLoadBalancingClient}
import com.amazonaws.services.s3.{AmazonS3, AmazonS3Client}

trait AmazonAutoScalingService {
  def autoScalingClient(credentials: AWSCredentialsProvider): AmazonAutoScaling = new AmazonAutoScalingClient(credentials)
}

trait AmazonCloudFormationService {
  def cloudFormationClient(credentials: AWSCredentialsProvider): AmazonCloudFormation = new AmazonCloudFormationClient(credentials)
}

trait AmazonElasticLoadBalancingService {
  def elasticLoadBalancingClient(credentials: AWSCredentialsProvider): AmazonElasticLoadBalancing = new AmazonElasticLoadBalancingClient(credentials)
}

trait AmazonS3Service {
  def s3Client(credentials: AWSCredentialsProvider): AmazonS3 = new AmazonS3Client(credentials)
} 

//设置package包名称以及导入依赖的类
package daos

import java.io.{File, FileOutputStream, InputStream}
import javax.inject.{Inject, Singleton}

import com.amazonaws.auth.DefaultAWSCredentialsProviderChain
import com.amazonaws.regions.Regions
import com.amazonaws.services.s3.{AmazonS3, AmazonS3ClientBuilder}
import org.apache.commons.io.IOUtils
import play.api.Configuration



@Singleton
class ApplicationStartupS3DAO @Inject()(config: Configuration) {
  private val s3: AmazonS3 = AmazonS3ClientBuilder.standard()
      .withCredentials(new DefaultAWSCredentialsProviderChain())
      .withRegion(Regions.EU_WEST_2)
      .build()
  private val bucketName = config.underlying.getString("s3-static")

  def downloadImageToTempFile(key: String): File = {
    val inputStream: InputStream = s3.getObject(bucketName, key).getObjectContent
    val tempFile = File.createTempFile(s"temp-file-$key", ".tmp")
    tempFile.deleteOnExit()
    val out = new FileOutputStream(tempFile)
    IOUtils.copy(inputStream, out)
    inputStream.close()
    out.close()
    tempFile
  }
} 

//设置package包名称以及导入依赖的类
package com.mobilerq.awsutil.s3

import com.amazonaws.services.s3.AmazonS3
import com.amazonaws.services.s3.model.{ListNextBatchOfObjectsRequest, ListObjectsRequest, ObjectListing}


class ListObjectIterator(request: ListObjectsRequest)(implicit client: AmazonS3) extends Iterator[ObjectListing] {

  def this(bucket: String)(implicit client: AmazonS3) =
    this(new ListObjectsRequest().withBucketName(bucket))

  def this(bucket: String, prefix: String)(implicit client: AmazonS3) =
    this(new ListObjectsRequest().withBucketName(bucket).withPrefix(prefix))

  var last: ObjectListing = _

  override def hasNext: Boolean = last == null || last.isTruncated

  override def next(): ObjectListing = {
    if (!hasNext) throw new IllegalStateException("empty iterator")
    last = if (last == null) {
      client.listObjects(request)
    } else {
      client.listNextBatchOfObjects(new ListNextBatchOfObjectsRequest(last)
        .withGeneralProgressListener[ListNextBatchOfObjectsRequest](request.getGeneralProgressListener)
        .withRequestMetricCollector[ListNextBatchOfObjectsRequest](request.getRequestMetricCollector)
        .withSdkClientExecutionTimeout[ListNextBatchOfObjectsRequest](Option(request.getSdkClientExecutionTimeout).map(_.intValue()).getOrElse(0))
        .withSdkRequestTimeout[ListNextBatchOfObjectsRequest](Option(request.getSdkRequestTimeout).map(_.intValue()).getOrElse(0))
      )
    }
    last
  }

} 

//设置package包名称以及导入依赖的类
package uk.co.telegraph.cloud.aws

import com.amazonaws.services.s3.transfer.{TransferManager, TransferManagerBuilder}
import com.amazonaws.services.s3.{AmazonS3, AmazonS3ClientBuilder}
import sbt.{File, Logger, URI}
import uk.co.telegraph.cloud.AuthCredentials

import AwsS3Bucket._

private [aws] trait AwsS3Bucket { this: AwsClientWithAuth =>


  lazy val s3Client: AmazonS3 = AmazonS3ClientBuilder.standard()
    .withRegion     ( region )
    .withCredentials( authProvider )
    .build()

  lazy val transferManager: TransferManager = TransferManagerBuilder.standard()
    .withS3Client(s3Client)
    .build()

  
  def pushTemplate(storageUrl:URI, localPath:File):Unit = {
    if( storageUrl.getScheme != "s3" ){
      throw S3InvalidProtocol
    }
    if( !localPath.exists() ){
      throw S3InvalidLocalPath
    }

    val s3Bucket = storageUrl.getHost
    val s3Key    = storageUrl.getPath.replaceFirst("^/", "")

    if (localPath.isDirectory) {
      transferManager.uploadDirectory(s3Bucket, s3Key, localPath, true).waitForCompletion()
    } else {
      transferManager.upload(s3Bucket, s3Key, localPath).waitForCompletion()
    }
  }
}

object AwsS3Bucket {
  object S3InvalidProtocol extends Exception{
    override def getMessage: String = "S3Client - Invalid protocol for S3 Path (s3://{s3Bucket}/{s3Key})."
  }

  object S3InvalidLocalPath extends Exception {
    override def getMessage: String = "S3Client - Invalid LocalPath - Path does not exist."
  }

  private case class AwsS3BucketImp(region:String, authCredentials: AuthCredentials, log: Logger)
    extends AwsS3Bucket with AwsClientWithAuth

  def apply(region:String, authCredentials: AuthCredentials)(implicit log:Logger):AwsS3Bucket = {
    AwsS3BucketImp(region, authCredentials, log)
  }
} 

//设置package包名称以及导入依赖的类
package services

import java.io.InputStream
import com.amazonaws.auth.profile.ProfileCredentialsProvider
import com.amazonaws.services.s3.{AmazonS3, AmazonS3Client}
import com.amazonaws.services.s3.model._
import play.api.Logger
import scala.util.{Try, Failure, Success}




object S3Service {

  val logger: Logger = Logger("S3ServiceLogger")
  lazy val s3Client: AmazonS3 = initialize()

  def initialize(): AmazonS3Client = {

    try {
      val s3Client: AmazonS3Client = new AmazonS3Client(new ProfileCredentialsProvider())
      logger.info("The process of getting credentials and creating s3Client object completed successfully")
      s3Client
    } catch {
      case e: Exception => {
        logger.error("An error occurred while getting credentials and creating s3Client object")
        throw e
      }
    }
  }


  def storeDataToAmazonS3(key: String, inputStream: InputStream): Try[Unit] = {

    try {
      val lenght: Int = inputStream.available()
      val objectMetadata: ObjectMetadata = new ObjectMetadata()
      objectMetadata.setContentLength(lenght)
      val request: PutObjectRequest = new PutObjectRequest(ConfigurationService.bucketName, key, inputStream, objectMetadata)
      s3Client.putObject(request)
      logger.info("Request was transmitted to Amazon S3 successfully")
      Success(Unit)

    } catch {
      case e: Exception =>{
        Failure(e)
      }
    }
  }
} 

//设置package包名称以及导入依赖的类
package uk.gov.homeoffice.aws.s3

import java.net.URL

import com.amazonaws.ClientConfiguration
import com.amazonaws.auth.AWSCredentials
import com.amazonaws.services.s3.model.{CryptoConfiguration, EncryptionMaterialsProvider}
import com.amazonaws.services.s3.{AmazonS3, AmazonS3Client, AmazonS3EncryptionClient}

trait S3Client extends AmazonS3

class S3PlainClient(endpoint: URL, credentials: AWSCredentials)(implicit clientConfiguration: ClientConfiguration = new ClientConfiguration) extends AmazonS3Client(credentials, clientConfiguration) with S3Client {
  setEndpoint(endpoint.toString)

  def clientConfig = clientConfiguration
}

class S3EncryptionClient(endpoint: URL, credentials: AWSCredentials,
                         encryptionMaterialsProvider: EncryptionMaterialsProvider, cryptoConfiguration: CryptoConfiguration)
                        (implicit clientConfiguration: ClientConfiguration = new ClientConfiguration)
  extends AmazonS3EncryptionClient(credentials, encryptionMaterialsProvider, clientConfiguration, cryptoConfiguration) with S3Client {
  setEndpoint(endpoint.toString)

  def clientConfig = clientConfiguration
} 

//设置package包名称以及导入依赖的类
package com.malliina.pics

import java.nio.file.Path

import akka.stream.scaladsl.StreamConverters
import com.amazonaws.regions.Regions
import com.amazonaws.services.s3.model.ObjectListing
import com.amazonaws.services.s3.{AmazonS3, AmazonS3ClientBuilder}
import com.malliina.storage.StorageLong

import scala.collection.JavaConversions._

class BucketFiles(val aws: AmazonS3, val bucket: BucketName) extends PicFiles {
  val bucketName: String = bucket.name

  override def load(from: Int, until: Int): Seq[Key] = {
    val size = until - from
    if (size <= 0) Nil
    else loadAcc(until, aws.listObjects(bucketName), Nil).drop(from)
  }

  private def loadAcc(desiredSize: Int, current: ObjectListing, acc: Seq[Key]): Seq[Key] = {
    val newAcc = acc ++ current.getObjectSummaries.map(s => Key(s.getKey))
    if (!current.isTruncated || newAcc.size >= desiredSize) newAcc take desiredSize
    else loadAcc(desiredSize, aws.listNextBatchOfObjects(current), newAcc)
  }

  override def contains(key: Key): Boolean =
    aws.doesObjectExist(bucketName, key.key)

  override def get(key: Key): DataStream = {
    val obj = aws.getObject(bucketName, key.key)
    val meta = obj.getObjectMetadata
    DataStream(
      StreamConverters.fromInputStream(obj.getObjectContent),
      Option(meta.getContentLength).map(_.bytes),
      Option(meta.getContentType).map(ContentType.apply)
    )
  }

  override def put(key: Key, file: Path): Unit =
    aws.putObject(bucketName, key.key, file.toFile)

  override def remove(key: Key): Unit =
    aws.deleteObject(bucketName, key.key)
}

object BucketFiles {
  def forBucket(bucket: BucketName) = forS3(Regions.EU_WEST_1, bucket)

  def forS3(region: Regions, bucket: BucketName): BucketFiles = {
    val bucketName = bucket.name
    val aws = AmazonS3ClientBuilder.standard().withRegion(region).build()
    if (!aws.doesBucketExist(bucketName))
      aws.createBucket(bucketName)
    new BucketFiles(aws, bucket)
  }
} 

//设置package包名称以及导入依赖的类
package utils

import java.io.{BufferedInputStream, InputStream}
import java.nio.file.{Files, Paths}

import com.amazonaws.services.s3.{AmazonS3, AmazonS3ClientBuilder}
import com.google.inject.name.Named
import com.google.inject.{AbstractModule, Provides, Singleton}
import play.api.Logger

class DigitalElementIndexModule extends AbstractModule {

  private[this] val fileUri = "^file:(.*)".r
  private[this] val s3Uri = "^s3://([^/]+)/(.*)".r

  override def configure(): Unit = ()

  @Provides
  @Named("DigitalElementIndex")
  @Singleton
  def getIndex(@javax.inject.Inject() environmentVariables: EnvironmentVariables): DigitalElementIndex = {
    val is: InputStream = environmentVariables.digitalElementFileUri match {
      case fileUri(path) => new BufferedInputStream(Files.newInputStream(Paths.get(path)))
      case s3Uri(bucket, key) => {
        val s3: AmazonS3 = AmazonS3ClientBuilder.standard().build()
        s3.getObject(bucket, key).getObjectContent()
      }
      case _ => throw new IllegalArgumentException("Invalid digitalElementFileUri.  Must use either s3:// or file:// protocol")
    }
    Logger.info(s"Building index from ${environmentVariables.digitalElementFileUri}")
    val start = System.currentTimeMillis()
    val index = DigitalElement.buildIndex(is, ';', '\n')
    is.close()
    Logger.info(s"Indexed ${index.size} records in ${(System.currentTimeMillis() - start) / 1000} seconds")
    index
  }

}