Python google.cloud方法代码示例

# 需要导入模块: import google [as 别名]
# 或者: from google import cloud [as 别名]
def copy_file_to(self, local_path_or_handle, remote_path, metadata=None):
    """Copy file from a local path to a remote path."""
    client = _storage_client()
    bucket_name, path = get_bucket_name_and_path(remote_path)

    try:
      bucket = client.bucket(bucket_name)
      blob = bucket.blob(path, chunk_size=self._chunk_size())
      if metadata:
        blob.metadata = metadata

      if isinstance(local_path_or_handle, basestring):
        blob.upload_from_filename(local_path_or_handle)
      else:
        blob.upload_from_file(local_path_or_handle, rewind=True)

    except google.cloud.exceptions.GoogleCloudError:
      logs.log_warn('Failed to copy local file %s to cloud storage file %s.' %
                    (local_path_or_handle, remote_path))
      raise

    return True 

# 需要导入模块: import google [as 别名]
# 或者: from google import cloud [as 别名]
def copy_blob(self, remote_source, remote_target):
    """Copy a remote file to another remote location."""
    source_bucket_name, source_path = get_bucket_name_and_path(remote_source)
    target_bucket_name, target_path = get_bucket_name_and_path(remote_target)

    client = _storage_client()
    try:
      source_bucket = client.bucket(source_bucket_name)
      source_blob = source_bucket.blob(source_path)
      target_bucket = client.bucket(target_bucket_name)
      source_bucket.copy_blob(source_blob, target_bucket, target_path)
    except google.cloud.exceptions.GoogleCloudError:
      logs.log_warn('Failed to copy cloud storage file %s to cloud storage '
                    'file %s.' % (remote_source, remote_target))
      raise

    return True 

# 需要导入模块: import google [as 别名]
# 或者: from google import cloud [as 别名]
def write_data(self, data, remote_path, metadata=None):
    """Write the data of a remote file."""
    client = _storage_client()
    bucket_name, path = get_bucket_name_and_path(remote_path)

    try:
      bucket = client.bucket(bucket_name)
      blob = bucket.blob(path, chunk_size=self._chunk_size())
      if metadata:
        blob.metadata = metadata
      blob.upload_from_string(data)
    except google.cloud.exceptions.GoogleCloudError:
      logs.log_warn('Failed to write cloud storage file %s.' % remote_path)
      raise

    return True 

# 需要导入模块: import google [as 别名]
# 或者: from google import cloud [as 别名]
def generate_life_cycle_config(action, age=None, num_newer_versions=None):
  """Generate GCS lifecycle management config.

  For the reference, see https://cloud.google.com/storage/docs/lifecycle and
  https://cloud.google.com/storage/docs/managing-lifecycles.
  """
  rule = {}
  rule['action'] = {'type': action}
  rule['condition'] = {}
  if age is not None:
    rule['condition']['age'] = age
  if num_newer_versions is not None:
    rule['condition']['numNewerVersions'] = num_newer_versions

  config = {'rule': [rule]}
  return config 

# 需要导入模块: import google [as 别名]
# 或者: from google import cloud [as 别名]
def _get_bucket(self):
        """get a bucket based on a bucket name. If it doesn't exist, create it.
        """

        # Case 1: The bucket already exists
        try:
            self._bucket = self._bucket_service.get_bucket(self._bucket_name)

        # Case 2: The bucket needs to be created
        except google.cloud.exceptions.NotFound:
            self._bucket = self._bucket_service.create_bucket(self._bucket_name)

        # Case 3: The bucket name is already taken
        except:
            bot.exit("Cannot get or create %s" % self._bucket_name)

        return self._bucket 

# 需要导入模块: import google [as 别名]
# 或者: from google import cloud [as 别名]
def _get_bucket(self, bucket_name):
        """get a bucket based on a bucket name. If it doesn't exist, create it.

           Parameters
           ==========
           bucket_name: the name of the bucket to get (or create). It should
                        not contain google, and should be all lowercase with -
                        or underscores.
        """

        # Case 1: The bucket already exists
        try:
            bucket = self._bucket_service.get_bucket(bucket_name)

        # Case 2: The bucket needs to be created
        except google.cloud.exceptions.NotFound:
            bucket = self._bucket_service.create_bucket(bucket_name)

        # Case 2: The bucket name is already taken
        except:
            bot.exit("Cannot get or create %s, is the name taken?" % bucket_name)

        return bucket 

# 需要导入模块: import google [as 别名]
# 或者: from google import cloud [as 别名]
def delete_blob(to_delete):
    # [START delete_blob]
    from google.cloud.exceptions import NotFound

    client = storage.Client()
    bucket = client.get_bucket("my-bucket")
    blobs = list(bucket.list_blobs())
    assert len(blobs) > 0
    # [<Blob: my-bucket, my-file.txt>]
    bucket.delete_blob("my-file.txt")
    try:
        bucket.delete_blob("doesnt-exist")
    except NotFound:
        pass
    # [END delete_blob]

    blob = None
    # [START delete_blobs]
    bucket.delete_blobs([blob], on_error=lambda blob: None)
    # [END delete_blobs]

    to_delete.append(bucket) 

# 需要导入模块: import google [as 别名]
# 或者: from google import cloud [as 别名]
def copy_file_from(self, remote_path, local_path):
    """Copy file from a remote path to a local path."""
    client = _storage_client()
    bucket_name, path = get_bucket_name_and_path(remote_path)

    try:
      bucket = client.bucket(bucket_name)
      blob = bucket.blob(path, chunk_size=self._chunk_size())
      blob.download_to_filename(local_path)
    except google.cloud.exceptions.GoogleCloudError:
      logs.log_warn('Failed to copy cloud storage file %s to local file %s.' %
                    (remote_path, local_path))
      raise

    return True 

# 需要导入模块: import google [as 别名]
# 或者: from google import cloud [as 别名]
def read_data(self, remote_path):
    """Read the data of a remote file."""
    bucket_name, path = get_bucket_name_and_path(remote_path)

    client = _storage_client()
    try:
      bucket = client.bucket(bucket_name)
      blob = bucket.blob(path, chunk_size=self._chunk_size())
      return blob.download_as_string()
    except google.cloud.exceptions.GoogleCloudError as e:
      if e.code == 404:
        return None

      logs.log_warn('Failed to read cloud storage file %s.' % remote_path)
      raise 

# 需要导入模块: import google [as 别名]
# 或者: from google import cloud [as 别名]
def get_bucket_name_and_path(cloud_storage_file_path):
  """Return bucket name and path given a full cloud storage path."""
  filtered_path = utils.strip_from_left(cloud_storage_file_path, GS_PREFIX)
  _, bucket_name_and_path = filtered_path.split('/', 1)

  if '/' in bucket_name_and_path:
    bucket_name, path = bucket_name_and_path.split('/', 1)
  else:
    bucket_name = bucket_name_and_path
    path = ''

  return bucket_name, path 

# 需要导入模块: import google [as 别名]
# 或者: from google import cloud [as 别名]
def exists(cloud_storage_file_path, ignore_errors=False):
  """Return whether if a cloud storage file exists."""
  try:
    return bool(_provider().get(cloud_storage_file_path))
  except HttpError:
    if not ignore_errors:
      logs.log_error('Failed when trying to find cloud storage file %s.' %
                     cloud_storage_file_path)

    return False 

# 需要导入模块: import google [as 别名]
# 或者: from google import cloud [as 别名]
def last_updated(cloud_storage_file_path):
  """Return last updated value by parsing stats for all blobs under a cloud
  storage path."""
  last_update = None
  for blob in _provider().list_blobs(cloud_storage_file_path):
    if not last_update or blob['updated'] > last_update:
      last_update = blob['updated']
  if last_update:
    # Remove UTC tzinfo to make these comparable.
    last_update = last_update.replace(tzinfo=None)
  return last_update 

# 需要导入模块: import google [as 别名]
# 或者: from google import cloud [as 别名]
def get_blobs(cloud_storage_path, recursive=True):
  """Return blobs under the given cloud storage path."""
  for blob in _provider().list_blobs(cloud_storage_path, recursive=recursive):
    yield blob 

# 需要导入模块: import google [as 别名]
# 或者: from google import cloud [as 别名]
def __init__(self, logger=None, destination=None, *args, **kwargs):
        import google
        from google.cloud import pubsub, pubsub_v1
        self.logger = logger
        if logger is None:
            self.logger = logging.getLogger('null-logger')
            self.logger.setLevel(9999)
        if destination == "full_ipv4":
            self.topic_url = os.environ.get('PUBSUB_IPV4_TOPIC_URL')
        elif destination == "alexa_top1mil":
            self.topic_url = os.environ.get('PUBSUB_ALEXA_TOPIC_URL')
        self.cert_topic_url = os.environ.get('PUBSUB_CERT_TOPIC_URL')
        if not self.topic_url:
            raise Exception('missing $PUBSUB_[IPV4|ALEXA]_TOPIC_URL')
        if not self.cert_topic_url:
            raise Exception('missing $PUBSUB_CERT_TOPIC_URL')
        batch_settings = pubsub_v1.types.BatchSettings(
            # "The entire request including one or more messages must
            #  be smaller than 10MB, after decoding."
            max_bytes=8192000,  # 8 MB
            max_latency=15,     # 15 seconds
        )
        self.publisher = pubsub.PublisherClient(batch_settings)
        self.publish_count = {}
        try:
            self.publisher.get_topic(self.topic_url)
            self.publisher.get_topic(self.cert_topic_url)
        except google.api_core.exceptions.GoogleAPICallError as e:
            logger.error(e.message)
            raise
        self._state = PubsubState() 

# 需要导入模块: import google [as 别名]
# 或者: from google import cloud [as 别名]
def download_to_file(to_delete):
    # [START download_to_file]
    from google.cloud.storage import Blob

    client = storage.Client(project="my-project")
    bucket = client.get_bucket("my-bucket")
    encryption_key = "c7f32af42e45e85b9848a6a14dd2a8f6"
    blob = Blob("secure-data", bucket, encryption_key=encryption_key)
    blob.upload_from_string("my secret message.")
    with open("/tmp/my-secure-file", "wb") as file_obj:
        blob.download_to_file(file_obj)
    # [END download_to_file]

    to_delete.append(blob)