Python PluralityCheckableIterator.is_empty方法代码示例

# 需要导入模块: from gslib.plurality_checkable_iterator import PluralityCheckableIterator [as 别名]
# 或者: from gslib.plurality_checkable_iterator.PluralityCheckableIterator import is_empty [as 别名]
  def _RecursePrint(self, blr):
    """
    Expands a bucket listing reference and recurses to its children, calling
    _PrintInfoAboutBucketListingRef for each expanded object found.

    Args:
      blr: An instance of BucketListingRef.

    Returns:
      Tuple containing (number of object, total number of bytes)
    """
    num_bytes = 0
    num_objs = 0

    if blr.HasKey():
      blr_iterator = iter([blr])
    elif blr.HasPrefix():
      blr_iterator = self.WildcardIterator(
          '%s/*' % blr.GetRStrippedUriString(), all_versions=self.all_versions)
    elif blr.NamesBucket():
      blr_iterator = self.WildcardIterator(
          '%s*' % blr.GetUriString(), all_versions=self.all_versions)
    else:
      # This BLR didn't come from a bucket listing. This case happens for
      # BLR's instantiated from a user-provided URI.
      blr_iterator = PluralityCheckableIterator(
          UriOnlyBlrExpansionIterator(
              self, blr, all_versions=self.all_versions))
      if blr_iterator.is_empty() and not ContainsWildcard(blr.GetUriString()):
        raise CommandException('No such object %s' % blr.GetUriString())

    for cur_blr in blr_iterator:
      if self.exclude_patterns:
        tomatch = cur_blr.GetUriString()
        skip = False
        for pattern in self.exclude_patterns:
          if fnmatch.fnmatch(tomatch, pattern):
            skip = True
            break
        if skip:
          continue
      if cur_blr.HasKey():
        # Object listing.
        no, nb = self._PrintInfoAboutBucketListingRef(cur_blr)
      else:
        # Subdir listing.
        if cur_blr.GetUriString().endswith('//'):
          # Expand gs://bucket// into gs://bucket//* so we don't infinite
          # loop. This case happens when user has uploaded an object whose
          # name begins with a /.
          cur_blr = BucketListingRef(self.suri_builder.StorageUri(
              '%s*' % cur_blr.GetUriString()), None, None, cur_blr.headers)
        no, nb = self._RecursePrint(cur_blr)
      num_bytes += nb
      num_objs += no

    if blr.HasPrefix() and not self.summary_only:
      self._PrintSummaryLine(num_bytes, blr.GetUriString().encode('utf-8'))

    return num_objs, num_bytes

# 需要导入模块: from gslib.plurality_checkable_iterator import PluralityCheckableIterator [as 别名]
# 或者: from gslib.plurality_checkable_iterator.PluralityCheckableIterator import is_empty [as 别名]
 def testPluralityCheckableIteratorWith3Elems(self):
   """Tests PluralityCheckableIterator with 3 elements."""
   input_list = range(3)
   it = iter(input_list)
   pcit = PluralityCheckableIterator(it)
   self.assertFalse(pcit.is_empty())
   self.assertTrue(pcit.has_plurality())
   output_list = list(pcit)
   self.assertEqual(input_list, output_list)

# 需要导入模块: from gslib.plurality_checkable_iterator import PluralityCheckableIterator [as 别名]
# 或者: from gslib.plurality_checkable_iterator.PluralityCheckableIterator import is_empty [as 别名]
 def __iter__(self):
   for blr in self.blr_iter:
     uri = blr.GetUri()
     if uri.names_object():
       # URI could be a bucket subdir.
       implicit_subdir_iterator = PluralityCheckableIterator(
           self.name_expansion_instance._WildcardIterator(
               self.name_expansion_instance.suri_builder.StorageUri(
                   '%s/%s' % (uri.uri.rstrip('/'),
                   self.name_expansion_instance._flatness_wildcard[
                       self.flat]))))
       if not implicit_subdir_iterator.is_empty():
         for exp_blr in implicit_subdir_iterator:
           yield (True, exp_blr)
       else:
         yield (False, blr)
     else:
       yield (False, blr)

# 需要导入模块: from gslib.plurality_checkable_iterator import PluralityCheckableIterator [as 别名]
# 或者: from gslib.plurality_checkable_iterator.PluralityCheckableIterator import is_empty [as 别名]
def NameExpansionIterator(command_name, proj_id_handler, headers, debug,
                          bucket_storage_uri_class, uri_strs,
                          recursion_requested,
                          have_existing_dst_container=None, flat=True):
  """
  Static factory function for instantiating _NameExpansionIterator, which
  wraps the resulting iterator in a PluralityCheckableIterator and checks
  that it is non-empty.

  Args are as documented in constructor for _NameExpansionIterator class.
  """
  name_expansion_iterator = _NameExpansionIterator(
      command_name, proj_id_handler, headers, debug, bucket_storage_uri_class,
      uri_strs, recursion_requested, have_existing_dst_container, flat)
  name_expansion_iterator = PluralityCheckableIterator(name_expansion_iterator)
  if name_expansion_iterator.is_empty():
    raise CommandException('No URIs matched')
  return name_expansion_iterator

# 需要导入模块: from gslib.plurality_checkable_iterator import PluralityCheckableIterator [as 别名]
# 或者: from gslib.plurality_checkable_iterator.PluralityCheckableIterator import is_empty [as 别名]
def NameExpansionIterator(command_name, proj_id_handler, headers, debug,
                          logger, bucket_storage_uri_class, uri_strs,
                          recursion_requested,
                          have_existing_dst_container=None, flat=True,
                          all_versions=False,
                          for_all_version_delete=False,
                          cmd_supports_recursion=True):
  """
  Static factory function for instantiating _NameExpansionIterator, which
  wraps the resulting iterator in a PluralityCheckableIterator and checks
  that it is non-empty. Also, allows uri_strs can be either an array or an
  iterator.

  Args:
    command_name: name of command being run.
    proj_id_handler: ProjectIdHandler to use for current command.
    headers: Dictionary containing optional HTTP headers to pass to boto.
    debug: Debug level to pass in to boto connection (range 0..3).
    logger: logging.Logger object.
    bucket_storage_uri_class: Class to instantiate for cloud StorageUris.
        Settable for testing/mocking.
    uri_strs: PluralityCheckableIterator of URI strings needing expansion.
    recursion_requested: True if -R specified on command-line.
    have_existing_dst_container: Bool indicator whether this is a copy
        request to an existing bucket, bucket subdir, or directory. Default
        None value should be used in cases where this is not needed (commands
        other than cp).
    flat: Bool indicating whether bucket listings should be flattened, i.e.,
        so the mapped-to results contain objects spanning subdirectories.
    all_versions: Bool indicating whether to iterate over all object versions.
    for_all_version_delete: Bool indicating whether this is for an all-version
        delete.
    cmd_supports_recursion: Bool indicating whether this command supports a '-R'
        flag. Useful for printing helpful error messages.

  Examples of ExpandWildcardsAndContainers with flat=True:
    - Calling with one of the uri_strs being 'gs://bucket' will enumerate all
      top-level objects, as will 'gs://bucket/' and 'gs://bucket/*'.
    - 'gs://bucket/**' will enumerate all objects in the bucket.
    - 'gs://bucket/abc' will enumerate all next-level objects under directory
      abc (i.e., not including subdirectories of abc) if gs://bucket/abc/*
      matches any objects; otherwise it will enumerate the single name
      gs://bucket/abc
    - 'gs://bucket/abc/**' will enumerate all objects under abc or any of its
      subdirectories.
    - 'file:///tmp' will enumerate all files under /tmp, as will
      'file:///tmp/*'
    - 'file:///tmp/**' will enumerate all files under /tmp or any of its
      subdirectories.

  Example if flat=False: calling with gs://bucket/abc/* lists matching objects
  or subdirs, but not sub-subdirs or objects beneath subdirs.

  Note: In step-by-step comments below we give examples assuming there's a
  gs://bucket with object paths:
    abcd/o1.txt
    abcd/o2.txt
    xyz/o1.txt
    xyz/o2.txt
  and a directory file://dir with file paths:
    dir/a.txt
    dir/b.txt
    dir/c/
  """
  uri_strs = PluralityCheckableIterator(uri_strs)
  name_expansion_iterator = _NameExpansionIterator(
      command_name, proj_id_handler, headers, debug, logger,
      bucket_storage_uri_class, uri_strs, recursion_requested,
      have_existing_dst_container, flat, all_versions=all_versions,
      for_all_version_delete=for_all_version_delete,
      cmd_supports_recursion=cmd_supports_recursion)
  name_expansion_iterator = PluralityCheckableIterator(name_expansion_iterator)
  if name_expansion_iterator.is_empty():
    raise CommandException('No URIs matched')
  return name_expansion_iterator

# 需要导入模块: from gslib.plurality_checkable_iterator import PluralityCheckableIterator [as 别名]
# 或者: from gslib.plurality_checkable_iterator.PluralityCheckableIterator import is_empty [as 别名]
  def _ExpandUriAndPrintInfo(self, uri, listing_style, should_recurse=False):
    """
    Expands wildcards and directories/buckets for uri as needed, and
    calls _PrintInfoAboutBucketListingRef() on each.

    Args:
      uri: StorageUri being listed.
      listing_style: ListingStyle enum describing type of output desired.
      should_recurse: bool indicator of whether to expand recursively.

    Returns:
      Tuple (number of matching objects, number of bytes across these objects).
    """
    # We do a two-level loop, with the outer loop iterating level-by-level from
    # blrs_to_expand, and the inner loop iterating the matches at the current
    # level, printing them, and adding any new subdirs that need expanding to
    # blrs_to_expand (to be picked up in the next outer loop iteration).
    blrs_to_expand = [BucketListingRef(uri)]
    num_objs = 0
    num_bytes = 0
    expanding_top_level = True
    printed_one = False
    num_expanded_blrs = 0
    while len(blrs_to_expand):
      if printed_one:
        print
      blr = blrs_to_expand.pop(0)
      if blr.HasKey():
        blr_iterator = iter([blr])
      elif blr.HasPrefix():
        # Bucket subdir from a previous iteration. Print "header" line only if
        # we're listing more than one subdir (or if it's a recursive listing),
        # to be consistent with the way UNIX ls works.
        if num_expanded_blrs > 1 or should_recurse:
          print '%s:' % blr.GetUriString().encode('utf-8')
          printed_one = True
        blr_iterator = self.WildcardIterator('%s/*' %
                                             blr.GetRStrippedUriString(),
                                             all_versions=self.all_versions)
      elif blr.NamesBucket():
        blr_iterator = self.WildcardIterator('%s*' % blr.GetUriString(),
                                             all_versions=self.all_versions)
      else:
        # This BLR didn't come from a bucket listing. This case happens for
        # BLR's instantiated from a user-provided URI.
        blr_iterator = PluralityCheckableIterator(
            _UriOnlyBlrExpansionIterator(
                self, blr, all_versions=self.all_versions))
        if blr_iterator.is_empty() and not ContainsWildcard(uri):
          raise CommandException('No such object %s' % uri)
      for cur_blr in blr_iterator:
        num_expanded_blrs = num_expanded_blrs + 1
        if cur_blr.HasKey():
          # Object listing.
          (no, nb) = self._PrintInfoAboutBucketListingRef(
              cur_blr, listing_style)
          num_objs += no
          num_bytes += nb
          printed_one = True
        else:
          # Subdir listing. If we're at the top level of a bucket subdir
          # listing don't print the list here (corresponding to how UNIX ls
          # dir just prints its contents, not the name followed by its
          # contents).
          if (expanding_top_level and not uri.names_bucket()) or should_recurse:
            if cur_blr.GetUriString().endswith('//'):
              # Expand gs://bucket// into gs://bucket//* so we don't infinite
              # loop. This case happens when user has uploaded an object whose
              # name begins with a /.
              cur_blr = BucketListingRef(self.suri_builder.StorageUri(
                  '%s*' % cur_blr.GetUriString()), None, None, cur_blr.headers)
            blrs_to_expand.append(cur_blr)
          # Don't include the subdir name in the output if we're doing a
          # recursive listing, as it will be printed as 'subdir:' when we get
          # to the prefix expansion, the next iteration of the main loop.
          else:
            if listing_style == ListingStyle.LONG:
              print '%-33s%s' % (
                  '', cur_blr.GetUriString().encode('utf-8'))
            else:
              print cur_blr.GetUriString().encode('utf-8')
      expanding_top_level = False
    return (num_objs, num_bytes)