Python PluralityCheckableIterator.next方法代码示例

# 需要导入模块: from gslib.plurality_checkable_iterator import PluralityCheckableIterator [as 别名]
# 或者: from gslib.plurality_checkable_iterator.PluralityCheckableIterator import next [as 别名]
  def testPluralityCheckableIteratorReadsAheadAsNeeded(self):
    """Tests that the PCI does not unnecessarily read new elements."""

    class IterTest(object):

      def __init__(self):
        self.position = 0

      def __iter__(self):
        return self

      def next(self):
        if self.position == 3:
          raise StopIteration()
        self.position += 1

    # IsEmpty and PeekException should retrieve only 1 element from the
    # underlying iterator.
    pcit = PluralityCheckableIterator(IterTest())
    pcit.IsEmpty()
    pcit.PeekException()
    self.assertEquals(pcit.orig_iterator.position, 1)
    # HasPlurality requires populating 2 elements into the iterator.
    pcit.HasPlurality()
    self.assertEquals(pcit.orig_iterator.position, 2)
    # next should yield already-populated elements without advancing the
    # iterator.
    pcit.next()  # Yields element 1
    self.assertEquals(pcit.orig_iterator.position, 2)
    pcit.next()  # Yields element 2
    self.assertEquals(pcit.orig_iterator.position, 2)
    pcit.next()  # Yields element 3
    self.assertEquals(pcit.orig_iterator.position, 3)
    try:
      pcit.next()  # Underlying iterator is empty
      self.fail('Expected StopIteration')
    except StopIteration:
      pass

# 需要导入模块: from gslib.plurality_checkable_iterator import PluralityCheckableIterator [as 别名]
# 或者: from gslib.plurality_checkable_iterator.PluralityCheckableIterator import next [as 别名]

#.........这里部分代码省略.........
    Returns:
      True/False.
    """
        # Note: This function is called from __iter__, which is called from the
        # Command.Apply driver. Thus, all checksum computation will be run in a
        # single thread, which is good (having multiple threads concurrently
        # computing checksums would thrash the disk).
        if src_size != dst_size:
            return False
        if self.compute_file_checksums:
            (src_crc32c, src_md5, dst_crc32c, dst_md5) = _ComputeNeededFileChecksums(
                self.logger, src_url_str, src_size, src_crc32c, src_md5, dst_url_str, dst_size, dst_crc32c, dst_md5
            )
        if src_md5 != _NA and dst_md5 != _NA:
            self.logger.debug("Comparing md5 for %s and %s", src_url_str, dst_url_str)
            return src_md5 == dst_md5
        if src_crc32c != _NA and dst_crc32c != _NA:
            self.logger.debug("Comparing crc32c for %s and %s", src_url_str, dst_url_str)
            return src_crc32c == dst_crc32c
        if not self._WarnIfMissingCloudHash(src_url_str, src_crc32c, src_md5):
            self._WarnIfMissingCloudHash(dst_url_str, dst_crc32c, dst_md5)
        # Without checksums to compare we depend only on basic size comparison.
        return True

    def __iter__(self):
        """Iterates over src/dst URLs and produces a _DiffToApply sequence.

    Yields:
      The _DiffToApply.
    """
        # Strip trailing slashes, if any, so we compute tail length against
        # consistent position regardless of whether trailing slashes were included
        # or not in URL.
        base_src_url_len = len(self.base_src_url.url_string.rstrip("/\\"))
        base_dst_url_len = len(self.base_dst_url.url_string.rstrip("/\\"))
        src_url_str = dst_url_str = None
        # Invariant: After each yield, the URLs in src_url_str, dst_url_str,
        # self.sorted_src_urls_it, and self.sorted_dst_urls_it are not yet
        # processed. Each time we encounter None in src_url_str or dst_url_str we
        # populate from the respective iterator, and we reset one or the other value
        # to None after yielding an action that disposes of that URL.
        while not self.sorted_src_urls_it.IsEmpty() or src_url_str is not None:
            if src_url_str is None:
                (src_url_str, src_size, src_crc32c, src_md5) = self._ParseTmpFileLine(self.sorted_src_urls_it.next())
                # Skip past base URL and normalize slashes so we can compare across
                # clouds/file systems (including Windows).
                src_url_str_to_check = _EncodeUrl(src_url_str[base_src_url_len:].replace("\\", "/"))
                dst_url_str_would_copy_to = copy_helper.ConstructDstUrl(
                    self.base_src_url,
                    StorageUrlFromString(src_url_str),
                    True,
                    True,
                    self.base_dst_url,
                    False,
                    self.recursion_requested,
                ).url_string
            if self.sorted_dst_urls_it.IsEmpty():
                # We've reached end of dst URLs, so copy src to dst.
                yield _DiffToApply(src_url_str, dst_url_str_would_copy_to, _DiffAction.COPY)
                src_url_str = None
                continue
            if not dst_url_str:
                (dst_url_str, dst_size, dst_crc32c, dst_md5) = self._ParseTmpFileLine(self.sorted_dst_urls_it.next())
                # Skip past base URL and normalize slashes so we can compare acros
                # clouds/file systems (including Windows).
                dst_url_str_to_check = _EncodeUrl(dst_url_str[base_dst_url_len:].replace("\\", "/"))

            if src_url_str_to_check < dst_url_str_to_check:
                # There's no dst object corresponding to src object, so copy src to dst.
                yield _DiffToApply(src_url_str, dst_url_str_would_copy_to, _DiffAction.COPY)
                src_url_str = None
            elif src_url_str_to_check > dst_url_str_to_check:
                # dst object without a corresponding src object, so remove dst if -d
                # option was specified.
                if self.delete_extras:
                    yield _DiffToApply(None, dst_url_str, _DiffAction.REMOVE)
                dst_url_str = None
            else:
                # There is a dst object corresponding to src object, so check if objects
                # match.
                if self._ObjectsMatch(
                    src_url_str, src_size, src_crc32c, src_md5, dst_url_str, dst_size, dst_crc32c, dst_md5
                ):
                    # Continue iterating without yielding a _DiffToApply.
                    pass
                else:
                    yield _DiffToApply(src_url_str, dst_url_str, _DiffAction.COPY)
                src_url_str = None
                dst_url_str = None

        # If -d option specified any files/objects left in dst iteration should be
        # removed.
        if not self.delete_extras:
            return
        if dst_url_str:
            yield _DiffToApply(None, dst_url_str, _DiffAction.REMOVE)
            dst_url_str = None
        for line in self.sorted_dst_urls_it:
            (dst_url_str, _, _, _) = self._ParseTmpFileLine(line)
            yield _DiffToApply(None, dst_url_str, _DiffAction.REMOVE)

# 需要导入模块: from gslib.plurality_checkable_iterator import PluralityCheckableIterator [as 别名]
# 或者: from gslib.plurality_checkable_iterator.PluralityCheckableIterator import next [as 别名]

#.........这里部分代码省略.........
    Returns:
      True/False.
    """
    # Note: This function is called from __iter__, which is called from the
    # Command.Apply driver. Thus, all checksum computation will be run in a
    # single thread, which is good (having multiple threads concurrently
    # computing checksums would thrash the disk).
    if src_size != dst_size:
      return False
    if self.compute_checksums:
      (src_crc32c, src_md5, dst_crc32c, dst_md5) = _ComputeNeededFileChecksums(
          self.logger, src_url_str, src_size, src_crc32c, src_md5, dst_url_str,
          dst_size, dst_crc32c, dst_md5)
    if src_md5 != _NA and dst_md5 != _NA:
      self.logger.debug('Comparing md5 for %s and %s', src_url_str, dst_url_str)
      return src_md5 == dst_md5
    if src_crc32c != _NA and dst_crc32c != _NA:
      self.logger.debug(
          'Comparing crc32c for %s and %s', src_url_str, dst_url_str)
      return src_crc32c == dst_crc32c
    if not self._WarnIfMissingCloudHash(src_url_str, src_crc32c, src_md5):
      self._WarnIfMissingCloudHash(dst_url_str, dst_crc32c, dst_md5)
    # Without checksums to compare we depend only on basic size comparison.
    return True

  def __iter__(self):
    """Iterates over src/dst URLs and produces a _DiffToApply sequence.

    Yields:
      The _DiffToApply.
    """
    # Strip trailing slashes, if any, so we compute tail length against
    # consistent position regardless of whether trailing slashes were included
    # or not in URL.
    base_src_url_len = len(self.base_src_url.GetUrlString().rstrip('/\\'))
    base_dst_url_len = len(self.base_dst_url.GetUrlString().rstrip('/\\'))
    src_url_str = dst_url_str = None
    # Invariant: After each yield, the URLs in src_url_str, dst_url_str,
    # self.sorted_src_urls_it, and self.sorted_dst_urls_it are not yet
    # processed. Each time we encounter None in src_url_str or dst_url_str we
    # populate from the respective iterator, and we reset one or the other value
    # to None after yielding an action that disposes of that URL.
    while not self.sorted_src_urls_it.IsEmpty() or src_url_str is not None:
      if src_url_str is None:
        (src_url_str, src_size, src_crc32c, src_md5) = self._ParseTmpFileLine(
            self.sorted_src_urls_it.next())
        # Skip past base URL and normalize slashes so we can compare across
        # clouds/file systems (including Windows).
        src_url_str_to_check = src_url_str[base_src_url_len:].replace('\\', '/')
        dst_url_str_would_copy_to = copy_helper.ConstructDstUrl(
            self.base_src_url, StorageUrlFromString(src_url_str), True, True,
            True, self.base_dst_url, False,
            self.recursion_requested).GetUrlString()
      if self.sorted_dst_urls_it.IsEmpty():
        # We've reached end of dst URLs, so copy src to dst.
        yield _DiffToApply(
            src_url_str, dst_url_str_would_copy_to, _DiffAction.COPY)
        src_url_str = None
        continue
      if not dst_url_str:
        (dst_url_str, dst_size, dst_crc32c, dst_md5) = (
            self._ParseTmpFileLine(self.sorted_dst_urls_it.next()))
        # Skip past base URL and normalize slashes so we can compare acros
        # clouds/file systems (including Windows).
        dst_url_str_to_check = dst_url_str[base_dst_url_len:].replace('\\', '/')

      if src_url_str_to_check < dst_url_str_to_check:
        # There's no dst object corresponding to src object, so copy src to dst.
        yield _DiffToApply(
            src_url_str, dst_url_str_would_copy_to, _DiffAction.COPY)
        src_url_str = None
      elif src_url_str_to_check > dst_url_str_to_check:
        # dst object without a corresponding src object, so remove dst if -d
        # option was specified.
        if self.delete_extras:
          yield _DiffToApply(None, dst_url_str, _DiffAction.REMOVE)
        dst_url_str = None
      else:
        # There is a dst object corresponding to src object, so check if objects
        # match.
        if self._ObjectsMatch(
            src_url_str, src_size, src_crc32c, src_md5,
            dst_url_str, dst_size, dst_crc32c, dst_md5):
          # Continue iterating without yielding a _DiffToApply.
          src_url_str = None
          dst_url_str = None
        else:
          yield _DiffToApply(src_url_str, dst_url_str, _DiffAction.COPY)
          dst_url_str = None

    # If -d option specified any files/objects left in dst iteration should be
    # removed.
    if not self.delete_extras:
      return
    if dst_url_str:
      yield _DiffToApply(None, dst_url_str, _DiffAction.REMOVE)
      dst_url_str = None
    for line in self.sorted_dst_urls_it:
      (dst_url_str, _, _, _) = self._ParseTmpFileLine(line)
      yield _DiffToApply(None, dst_url_str, _DiffAction.REMOVE)