Java AmazonS3URI类代码示例

import com.amazonaws.services.s3.AmazonS3URI; //导入依赖的package包/类
private String regionForUri(AmazonS3 client, AmazonS3URI uri) {
  String bucketRegion = client.getBucketLocation(uri.getBucket());
  Region region = Region.fromValue(bucketRegion);

  // S3 doesn't have a US East 1 region, US East 1 is really the region
  // US Standard. US Standard places the data in either an east coast
  // or west coast data center geographically closest to you.
  // SigV4 requires you to mention a region while signing a request
  // and for the S3's US standard endpoints the value to be used is "us-east-1"
  // US West 1 has an endpoint and so is treated as a stand alone region,
  // US East 1 doesn't and so is bundled into US Standard
  if (region.equals(Region.US_Standard)) {
    bucketRegion = "us-east-1";
  } else {
    bucketRegion = region.toString();
  }
  return bucketRegion;
}
 

import com.amazonaws.services.s3.AmazonS3URI; //导入依赖的package包/类
private void startAllCopyJobs() {
  AmazonS3URI sourceBase = toAmazonS3URI(sourceBaseLocation.toUri());
  AmazonS3URI targetBase = toAmazonS3URI(replicaLocation.toUri());
  srcClient = s3ClientFactory.newInstance(sourceBase, s3s3CopierOptions);
  targetClient = s3ClientFactory.newInstance(targetBase, s3s3CopierOptions);
  transferManager = transferManagerFactory.newInstance(targetClient, s3s3CopierOptions);
  if (sourceSubLocations.isEmpty()) {
    copy(sourceBase, targetBase);
  } else {
    for (Path path : sourceSubLocations) {
      AmazonS3URI subLocation = toAmazonS3URI(path.toUri());
      String partitionKey = StringUtils.removeStart(subLocation.getKey(), sourceBase.getKey());
      partitionKey = StringUtils.removeStart(partitionKey, "/");
      AmazonS3URI targetS3Uri = toAmazonS3URI(new Path(replicaLocation, partitionKey).toUri());
      LOG.debug("Starting copyJob from {} to {}", subLocation, targetS3Uri);
      copy(subLocation, targetS3Uri);
    }
  }
}
 

import com.amazonaws.services.s3.AmazonS3URI; //导入依赖的package包/类
@Override
public Stream<String> lines() {
	LOGGER.debug("starting download from {}", uri);
	AmazonS3URI s3URI = new AmazonS3URI(uri);
	S3Object s3Object = s3Client.getObject(s3URI.getBucket(), s3URI.getKey());

	InputStream stream = s3Object.getObjectContent();
	return new BufferedReader(new InputStreamReader(stream)).lines();
}
 

import com.amazonaws.services.s3.AmazonS3URI; //导入依赖的package包/类
/**
 * A method that seeks and downloads the index for the set BAM URI.
 * Seeks an index file with the same name in the BAM directory
 * in case there's no custom index URI specified
 *
 * @param bamURI an http address of the required file.
 * @return A SeekableStream optional on index file URI
 */
Optional<SeekableStream> loadIndex(AmazonS3URI bamURI) throws IOException {
    LOG.info("Trying to set index file for " + bamURI.toString());
    Optional<AmazonS3URI> index = providedIndexURI()
            .map(Optional::of)
            .orElseGet(() -> nearbyIndexURI(bamURI));

    if (!index.isPresent()) {
        LOG.info("Index wasn't provided for " + bamURI.toString());
        return Optional.empty();
    }

    LOG.info("Start download index: " + index.get());
    AmazonS3URI indexURI = index.get();
    S3InputStreamFactory streamFactory = new S3InputStreamFactory(client);
    InputStream stream = streamFactory.loadFully(indexURI);
    long fileSize = client.getFileSize(indexURI);
    byte[] buffer = IOUtils.toByteArray(stream);

    if (fileSize != buffer.length) {
        throw new IOException("Failed to fully download index " + indexURI);
    }

    LOG.info("Finished download index: " + index.get());
    return Optional.of(new SeekableMemoryStream(buffer, indexURI.toString()));
}
 

import com.amazonaws.services.s3.AmazonS3URI; //导入依赖的package包/类
/**
 * A method that creates a SamReader object that's passed on to the HTSJDK.
 * Each time someone tries to open a SamReader on an URL,
 * HTSJDK checks if there's a custom reader factory and if it's there, this method is called.
 *
 * @param url target file URL
 * @return A SamReader object on a specified file URL
 */
@Override
public SamReader open(URL url) {
    PerformanceMonitor.start();
    AmazonS3URI amazonURI = new AmazonS3URI(url.toString());
    S3Client client = new S3Client();
    S3InputStreamFactory streamFactory = new S3InputStreamFactory(client);

    //download index file if is possible, and then start download .bam file
    final Optional<SeekableStream> indexStream;
    try {
        IndexLoader loader = new IndexLoader(client);
        indexStream = loader.loadIndex(amazonURI);
    } catch (IOException e) {
        throw new RuntimeIOException(e.getMessage() + " failed to download index", e);
    }

    SeekableStream stream = new S3SeekableStream(amazonURI, client, streamFactory);
    SamReaderFactory factory = SamReaderFactory.makeDefault();
    SamInputResource inputResource = SamInputResource.of(stream);

    indexStream.ifPresent(inputResource::index);

    return factory.open(inputResource);
}
 

import com.amazonaws.services.s3.AmazonS3URI; //导入依赖的package包/类
/**
 * A method that returns true if a correct s3 URI was provided and false otherwise.
 *
 * @param uri The provided URI for the file.
 * @return a boolean value that shows whether the correct URI was provided
 */
boolean isFileExisting(AmazonS3URI uri) {

    boolean exist = true;

    try {
        aws.getObjectMetadata(uri.getBucket(), uri.getKey());
    } catch (AmazonS3Exception e) {
        if (e.getStatusCode() == HttpStatus.SC_FORBIDDEN
                || e.getStatusCode() == HttpStatus.SC_NOT_FOUND) {
            exist = false;
        } else {
            throw e;
        }
    }
    return exist;
}
 

import com.amazonaws.services.s3.AmazonS3URI; //导入依赖的package包/类
public static void mockPrimitiveLoadFromTo(S3InputStreamFactory factory, int dataSize) {
    PowerMockito.mockStatic(S3InputStreamFactory.class);
    PowerMockito.when(factory.loadFromTo(Mockito.any(AmazonS3URI.class), Mockito.anyLong(), Mockito.anyLong()))
            .then((invocation) -> new InputStream() {
                        final Log log = Log.getInstance(InputStream.class);
                        private int size = dataSize;

                        @Override
                        public int read() throws IOException {
                            final int returnByte = (size-- > 0) ? FROM_STREAM_CHAR : -1;
                            log.debug("Stream(", this.hashCode(), " size = ", size, " return: ", returnByte);
                            return returnByte;
                        }
                    }
            );
}
 

import com.amazonaws.services.s3.AmazonS3URI; //导入依赖的package包/类
public static void mockAutoSeqLoadFromTo(S3InputStreamFactory factory) {
    PowerMockito.mockStatic(S3InputStreamFactory.class);
    PowerMockito
            .when(factory.loadFromTo(
                    Mockito.any(AmazonS3URI.class),
                    Mockito.anyLong(),
                    Mockito.anyLong()))
            .then((invocation) -> new InputStream() {

                        long from = (Long) invocation.getArguments()[1];
                        final long to = (Long) invocation.getArguments()[2];

                        @Override
                        public int read() throws IOException {
                            return (from < to) ? (int) from++ : END_BYTE;
                        }
                    }
            );
}
 

import com.amazonaws.services.s3.AmazonS3URI; //导入依赖的package包/类
@Override
public void setConf(AbstractConfig config) {
  this.config = (GeoIpOperationConfig) config;
  AmazonS3Client client = this.s3Factory.newInstance();

  AmazonS3URI uri = new AmazonS3URI(this.config.getGeoLiteDb());
  GetObjectRequest req = new GetObjectRequest(uri.getBucket(), uri.getKey());
  S3Object obj = client.getObject(req);

  try {
    this.databaseReader =
        new DatabaseReader.Builder(obj.getObjectContent()).withCache(new CHMCache()).build();
  } catch (IOException e) {
    throw new ConfigurationException("Unable to read " + this.config.getGeoLiteDb(), e);
  }
}
 

import com.amazonaws.services.s3.AmazonS3URI; //导入依赖的package包/类
public static BenderConfig load(AmazonS3ClientFactory s3ClientFactory, AmazonS3URI s3Uri) {
  AmazonS3Client s3 = s3ClientFactory.newInstance();
  S3Object s3object = s3.getObject(s3Uri.getBucket(), s3Uri.getKey());

  StringWriter writer = new StringWriter();

  try {
    IOUtils.copy(s3object.getObjectContent(), writer, "UTF-8");
  } catch (IOException e) {
    throw new ConfigurationException("Unable to read file from s3", e);
  }
  BenderConfig config = load(s3Uri.getKey().toString(), writer.toString());
  config.setConfigFile(s3Uri.getURI().toString());

  return config;
}
 

import com.amazonaws.services.s3.AmazonS3URI; //导入依赖的package包/类
/**
 * Parse string representing an AWS S3 URI
 */
protected void parse(String s3UriStr) {
	s3uri = new AmazonS3URI(s3UriStr);
	bucketName = s3uri.getBucket();
	key = s3uri.getKey();

	// Parse and set region
	String regionStr = s3uri.getRegion();
	try {
		//			if (regionStr == null) regionStr = Config.get().getString(Config.AWS_REGION, DEFAULT_AWS_REGION);
		//			region = Region.getRegion(Regions.valueOf(regionStr.toUpperCase()));
		if (regionStr != null) region = Region.getRegion(Regions.valueOf(regionStr.toUpperCase()));
	} catch (Exception e) {
		throw new RuntimeException("Cannot parse AWS region '" + regionStr + "'", e);
	}
}
 

import com.amazonaws.services.s3.AmazonS3URI; //导入依赖的package包/类
public BucketKey(String inputFilePath, String outputFilePath){
    AmazonS3URI srcURI = new AmazonS3URI(inputFilePath);
    AmazonS3URI destURI = new AmazonS3URI(outputFilePath);
    this.srcBucket = srcURI.getBucket();
    this.srcKey = srcURI.getKey();
    this.destBucket = destURI.getBucket();
    this.destPrefix = destURI.getKey();
}
 

import com.amazonaws.services.s3.AmazonS3URI; //导入依赖的package包/类
/**
 * Reads a file from S3 into a String object
 * @param s3Uri (eg. s3://bucket/file.ext)
 * @return String containing the content of the file in S3
 * @throws IOException if error reading file
 */
public String readFileFromS3(String s3Uri) throws IOException {
    AmazonS3URI s3FileUri = new AmazonS3URI(s3Uri);
    S3Object s3object = amazonS3Client.getObject(new GetObjectRequest(s3FileUri.getBucket(), s3FileUri.getKey()));
    
    return IOUtils.toString(s3object.getObjectContent());
}
 

import com.amazonaws.services.s3.AmazonS3URI; //导入依赖的package包/类
public static AmazonS3URI toAmazonS3URI(URI uri) {
  if (FS_PROTOCOL_S3.equalsIgnoreCase(uri.getScheme())) {
    return new AmazonS3URI(uri);
  } else if (S3Schemes.isS3Scheme(uri.getScheme())) {
    try {
      return new AmazonS3URI(new URI(FS_PROTOCOL_S3, uri.getUserInfo(), uri.getHost(), uri.getPort(), uri.getPath(),
          uri.getQuery(), uri.getFragment()));
    } catch (URISyntaxException e) {
      // ignore it, it will fail on the return
    }
  }
  // Build it anyway we'll get AmazonS3URI exception back.
  return new AmazonS3URI(uri);
}
 

import com.amazonaws.services.s3.AmazonS3URI; //导入依赖的package包/类
@Override
public AmazonS3 newInstance(AmazonS3URI uri, S3S3CopierOptions s3s3CopierOptions) {
  LOG.debug("trying to get a client for uri '{}'", uri);
  AmazonS3 globalClient = newGlobalInstance(s3s3CopierOptions);
  try {
    String bucketRegion = regionForUri(globalClient, uri);
    LOG.debug("Bucket region: {}", bucketRegion);
    return newInstance(bucketRegion, s3s3CopierOptions);
  } catch (IllegalArgumentException e) {
    LOG.warn("Using global (non region specific) client", e);
    return globalClient;
  }
}