Python RangeSet.union方法代码示例

# 需要导入模块: from rangelib import RangeSet [as 别名]
# 或者: from rangelib.RangeSet import union [as 别名]
  def AssertSequenceGood(self):
    # Simulate the sequences of transfers we will output, and check that:
    # - we never read a block after writing it, and
    # - we write every block we care about exactly once.

    # Start with no blocks having been touched yet.
    touched = RangeSet()

    # Imagine processing the transfers in order.
    for xf in self.transfers:
      # Check that the input blocks for this transfer haven't yet been touched.

      x = xf.src_ranges
      if self.version >= 2:
        for _, sr in xf.use_stash:
          x = x.subtract(sr)

      assert not touched.overlaps(x)
      # Check that the output blocks for this transfer haven't yet been touched.
      assert not touched.overlaps(xf.tgt_ranges)
      # Touch all the blocks written by this transfer.
      touched = touched.union(xf.tgt_ranges)

    # Check that we've written every target block.
    assert touched == self.tgt.care_map

# 需要导入模块: from rangelib import RangeSet [as 别名]
# 或者: from rangelib.RangeSet import union [as 别名]
 def AssertPartition(total, seq):
   """Assert that all the RangeSets in 'seq' form a partition of the
   'total' RangeSet (ie, they are nonintersecting and their union
   equals 'total')."""
   so_far = RangeSet()
   for i in seq:
     assert not so_far.overlaps(i)
     so_far = so_far.union(i)
   assert so_far == total

# 需要导入模块: from rangelib import RangeSet [as 别名]
# 或者: from rangelib.RangeSet import union [as 别名]
  def test_ValidateFileConsistency_incompleteRange(self):
    input_tmp = common.MakeTempDir()
    os.mkdir(os.path.join(input_tmp, 'IMAGES'))
    system_image = os.path.join(input_tmp, 'IMAGES', 'system.img')
    system_root = os.path.join(input_tmp, "SYSTEM")
    os.mkdir(system_root)

    # Write the test file that contain multiple blocks of zeros, and these
    # zero blocks will be omitted by kernel. And the test files will occupy one
    # block range each in the final system image.
    with open(os.path.join(system_root, 'a'), 'w') as f:
      f.write("aaa")
      f.write('\0' * 4096 * 3)
    with open(os.path.join(system_root, 'b'), 'w') as f:
      f.write("bbb")
      f.write('\0' * 4096 * 3)

    raw_file_map = os.path.join(input_tmp, 'IMAGES', 'raw_system.map')
    self._generate_system_image(system_image, system_root, raw_file_map)

    # Parse the generated file map and update the block ranges for each file.
    file_map_list = {}
    image_ranges = RangeSet()
    with open(raw_file_map, 'r') as f:
      for line in f.readlines():
        info = line.split()
        self.assertEqual(2, len(info))
        image_ranges = image_ranges.union(RangeSet(info[1]))
        file_map_list[info[0]] = RangeSet(info[1])

    # Add one unoccupied block as the shared block for all test files.
    mock_shared_block = RangeSet("10-20").subtract(image_ranges).first(1)
    with open(os.path.join(input_tmp, 'IMAGES', 'system.map'), 'w') as f:
      for key in sorted(file_map_list.keys()):
        line = "{} {}\n".format(
            key, file_map_list[key].union(mock_shared_block))
        f.write(line)

    # Prepare for the target zip file
    input_file = common.MakeTempFile()
    all_entries = ['SYSTEM/', 'SYSTEM/b', 'SYSTEM/a', 'IMAGES/',
                   'IMAGES/system.map', 'IMAGES/system.img']
    with zipfile.ZipFile(input_file, 'w') as input_zip:
      for name in all_entries:
        input_zip.write(os.path.join(input_tmp, name), arcname=name)

    input_zip = zipfile.ZipFile(input_file, 'r')
    info_dict = {'extfs_sparse_flag': '-s'}

    # Expect the validation to pass and both files are skipped due to
    # 'incomplete' block range.
    ValidateFileConsistency(input_zip, input_tmp, info_dict)

# 需要导入模块: from rangelib import RangeSet [as 别名]
# 或者: from rangelib.RangeSet import union [as 别名]

#.........这里部分代码省略.........
    total = 0

    stashes = {}
    stashed_blocks = 0
    max_stashed_blocks = 0

    free_stash_ids = []
    next_stash_id = 0

    for xf in self.transfers:

      if self.version < 2:
        assert not xf.stash_before
        assert not xf.use_stash

      for s, sr in xf.stash_before:
        assert s not in stashes
        if free_stash_ids:
          sid = heapq.heappop(free_stash_ids)
        else:
          sid = next_stash_id
          next_stash_id += 1
        stashes[s] = sid
        if self.version == 2:
          stashed_blocks += sr.size()
          out.append("stash %d %s\n" % (sid, sr.to_string_raw()))
        else:
          sh = self.HashBlocks(self.src, sr)
          if sh in stashes:
            stashes[sh] += 1
          else:
            stashes[sh] = 1
            stashed_blocks += sr.size()
            self.touched_src_ranges = self.touched_src_ranges.union(sr)
            out.append("stash %s %s\n" % (sh, sr.to_string_raw()))

      if stashed_blocks > max_stashed_blocks:
        max_stashed_blocks = stashed_blocks

      free_string = []
      free_size = 0

      if self.version == 1:
        src_str = xf.src_ranges.to_string_raw() if xf.src_ranges else ""
      elif self.version >= 2:

        #   <# blocks> <src ranges>
        #     OR
        #   <# blocks> <src ranges> <src locs> <stash refs...>
        #     OR
        #   <# blocks> - <stash refs...>

        size = xf.src_ranges.size()
        src_str = [str(size)]

        unstashed_src_ranges = xf.src_ranges
        mapped_stashes = []
        for s, sr in xf.use_stash:
          # TODO: We don't need 'sid' (nor free_stash_ids) in BBOTA v3+.
          sid = stashes.pop(s)
          unstashed_src_ranges = unstashed_src_ranges.subtract(sr)
          sh = self.HashBlocks(self.src, sr)
          sr = xf.src_ranges.map_within(sr)
          mapped_stashes.append(sr)
          if self.version == 2:
            src_str.append("%d:%s" % (sid, sr.to_string_raw()))

# 需要导入模块: from rangelib import RangeSet [as 别名]
# 或者: from rangelib.RangeSet import union [as 别名]
    def AddTransfer(tgt_name, src_name, tgt_ranges, src_ranges, style, by_id,
                    split=False):
      """Wrapper function for adding a Transfer()."""

      # We specialize diff transfers only (which covers bsdiff/imgdiff/move);
      # otherwise add the Transfer() as is.
      if style != "diff" or not split:
        Transfer(tgt_name, src_name, tgt_ranges, src_ranges, style, by_id)
        return

      # Handle .odex files specially to analyze the block-wise difference. If
      # most of the blocks are identical with only few changes (e.g. header),
      # we will patch the changed blocks only. This avoids stashing unchanged
      # blocks while patching. We limit the analysis to files without size
      # changes only. This is to avoid sacrificing the OTA generation cost too
      # much.
      if (tgt_name.split(".")[-1].lower() == 'odex' and
          tgt_ranges.size() == src_ranges.size()):

        # 0.5 threshold can be further tuned. The tradeoff is: if only very
        # few blocks remain identical, we lose the opportunity to use imgdiff
        # that may have better compression ratio than bsdiff.
        crop_threshold = 0.5

        tgt_skipped = RangeSet()
        src_skipped = RangeSet()
        tgt_size = tgt_ranges.size()
        tgt_changed = 0
        for src_block, tgt_block in zip(src_ranges.next_item(),
                                        tgt_ranges.next_item()):
          src_rs = RangeSet(str(src_block))
          tgt_rs = RangeSet(str(tgt_block))
          if self.src.ReadRangeSet(src_rs) == self.tgt.ReadRangeSet(tgt_rs):
            tgt_skipped = tgt_skipped.union(tgt_rs)
            src_skipped = src_skipped.union(src_rs)
          else:
            tgt_changed += tgt_rs.size()

          # Terminate early if no clear sign of benefits.
          if tgt_changed > tgt_size * crop_threshold:
            break

        if tgt_changed < tgt_size * crop_threshold:
          assert tgt_changed + tgt_skipped.size() == tgt_size
          print('%10d %10d (%6.2f%%) %s' % (tgt_skipped.size(), tgt_size,
                tgt_skipped.size() * 100.0 / tgt_size, tgt_name))
          AddSplitTransfers(
              "%s-skipped" % (tgt_name,),
              "%s-skipped" % (src_name,),
              tgt_skipped, src_skipped, style, by_id)

          # Intentionally change the file extension to avoid being imgdiff'd as
          # the files are no longer in their original format.
          tgt_name = "%s-cropped" % (tgt_name,)
          src_name = "%s-cropped" % (src_name,)
          tgt_ranges = tgt_ranges.subtract(tgt_skipped)
          src_ranges = src_ranges.subtract(src_skipped)

          # Possibly having no changed blocks.
          if not tgt_ranges:
            return

      # Add the transfer(s).
      AddSplitTransfers(
          tgt_name, src_name, tgt_ranges, src_ranges, style, by_id)