Python DVIDNodeService.get_roi_partition方法代码示例

# 需要导入模块: from libdvid import DVIDNodeService [as 别名]
# 或者: from libdvid.DVIDNodeService import get_roi_partition [as 别名]
    def test_roi_partition(self):
        node_service = DVIDNodeService(TEST_DVID_SERVER, self.uuid)
        node_service.create_roi("test_roi_partition")
        
        blocks = [(0,0,0),(1,1,1),(2,2,2),(3,3,3)]
        node_service.post_roi("test_roi_partition", blocks)
        substacks, packing_factor = node_service.get_roi_partition("test_roi_partition", 4)
        
        self.assertEqual(substacks, [SubstackXYZ(0,0,0,4*32)])        
        self.assertEqual( packing_factor, float(len(blocks))/(len(substacks) * 4**3) )

        blocks += [(4,0,0)]
        node_service.post_roi("test_roi_partition", blocks)
        substacks, packing_factor = node_service.get_roi_partition("test_roi_partition", 4)
 
        self.assertEqual(substacks, [SubstackXYZ(0,0,0,4*32), SubstackXYZ(128,0,0,4*32)])

# 需要导入模块: from libdvid import DVIDNodeService [as 别名]
# 或者: from libdvid.DVIDNodeService import get_roi_partition [as 别名]
def download_to_h5( hostname, uuid, instance, roi, output_filepath, dset_name=None, compression='lzf', overlap_px=0):
    """
    """
    ns = DVIDNodeService(hostname, uuid)
    va = VoxelsAccessor(hostname, uuid, instance, throttle=True)
    
    dset_name = dset_name or instance

    assert roi, "Must provide a ROI"
    logger.info("Downloading {hostname}/api/node/{uuid}/{instance}?roi={roi} to {output_filepath}/{dset_name}".format(**locals()))

    substacks, _packing_factor = ns.get_roi_partition(roi, SUBSTACK_SIZE / DVID_BLOCK_SIZE)

    # Substack tuples are (size, z, y, x)
    substacks_zyx = np.array(substacks)[:, 1:]
    
    # If the user specified an 'overlap', we add it to all substacks.
    # Technically, this isn't very efficient, because a lot of overlapping
    # pixels on the interior of the ROI will be fetched twice.
    substacks_zyx[:,0] -= overlap_px
    substacks_zyx[:,1] += overlap_px

    roi_bb = ( np.min(substacks_zyx, axis=0),
               np.max(substacks_zyx, axis=0)+SUBSTACK_SIZE )
    
    with h5py.File(output_filepath, 'a') as output_file:
        try:
            del output_file[dset_name]
        except KeyError:
            pass
        
        dset = output_file.create_dataset( dset_name, shape=roi_bb[1], dtype=va.dtype, chunks=True, compression=compression )
    
        for i, substack_zyx in enumerate(substacks_zyx):
            logger.info("Substack {}/{} {}: Downloading...".format( i, len(substacks_zyx), list(substack_zyx) ))
            
            # Append a singleton channel axis
            substack_bb = np.array(( tuple(substack_zyx) + (0,),
                                     tuple(substack_zyx + SUBSTACK_SIZE) + (1,) ))
            
            # Includes singleton channel
            substack_data = va.get_ndarray(*substack_bb)

            logger.info("Substack {}/{} {}: Writing...".format( i, len(substacks_zyx), list(substack_zyx) ))
            dset[bb_to_slicing(*substack_bb[:,:-1])] = substack_data[...,0]

    logger.info("DONE Downloading {hostname}/api/node/{uuid}/{instance}?roi={roi} to {output_filepath}/{dset_name}".format(**locals()))

# 需要导入模块: from libdvid import DVIDNodeService [as 别名]
# 或者: from libdvid.DVIDNodeService import get_roi_partition [as 别名]
    def parallelize_roi(self, roi, chunk_size, border=0, find_neighbors=False):
        """Creates an RDD from subvolumes found in an ROI.

        This is analogous to the Spark parallelize function.
        It currently defines the number of partitions as the 
        number of subvolumes.

        TODO: implement general partitioner given other
        input such as bounding box coordinates.
        
        Args:
            roi (str): name of DVID ROI at current server and uuid
            chunk_size (int): the desired dimension of the subvolume
            border (int): size of the border surrounding the subvolume
            find_neighbors (bool): whether to identify neighbors

        Returns:
            RDD as [(subvolume id, subvolume)] and # of subvolumes

        """

        # function will export and should include dependencies
        subvolumes = []  # x,y,z,x2,y2,z2

        from libdvid import DVIDNodeService, SubstackXYZ

        # extract roi for a given chunk size
        node_service = DVIDNodeService(str(self.dvid_server), str(self.uuid))
        substacks, packing_factor = node_service.get_roi_partition(str(roi), chunk_size / self.BLK_SIZE)

        # create roi array giving unique substack ids
        for substack_id, substack in enumerate(substacks):
            # use substack id as key
            subvolumes.append((substack_id, Subvolume(substack_id, substack, chunk_size, border)))

        # grab all neighbors for each substack
        if find_neighbors:
            # inefficient search for all boundaries
            for i in range(0, len(subvolumes) - 1):
                for j in range(i + 1, len(subvolumes)):
                    subvolumes[i][1].recordborder(subvolumes[j][1])

        # Potential TODO: custom partitioner for grouping close regions
        return self.sc.parallelize(subvolumes, len(subvolumes))

# 需要导入模块: from libdvid import DVIDNodeService [as 别名]
# 或者: from libdvid.DVIDNodeService import get_roi_partition [as 别名]
def copy_voxels( source_details,
                 destination_details,
                 transfer_cube_width_px=512,
                 roi=None,
                 subvol_bounds_zyx=None ):
    """
    Transfer voxels data from one DVID server to another.
    
    source_details:
        Either a tuple of (hostname, uuid, instance),
        or a url of the form http://hostname/api/node/uuid/instance
    
    destination_details:
        Same format as source_details, or just an instance name
        (in which case the destination is presumed to be in the same host/node as the source).
    
    transfer_cube_width_px:
        The data will be transferred one 'substack' at a time, with the given substack width.
    
    NOTE: Exactly ONE of the following parameters should be provided.
    
    roi:
        Same format as destination_details, but should point to a ROI instance.
    
    subvol_bounds_zyx:
        A tuple (start_zyx, stop_zyx) indicating a rectangular region to copy (instead of a ROI).
        Specified in pixel coordinates. Must be aligned to DVID block boundaries.
        For example: ((0,0,0), (1024, 1024, 512))
    """
    if isinstance(source_details, str):
        source_details = parse_instance_url( source_details )
    else:
        source_details = InstanceDetails(*source_details)
    src_accessor = VoxelsAccessor( *source_details )
    
    if isinstance(destination_details, str):
        destination_details = str_to_details( destination_details, default=source_details )
    else:
        destination_details = InstanceDetails(*destination_details)
    dest_accessor = VoxelsAccessor( *destination_details )

    assert (roi is not None) ^ (subvol_bounds_zyx is not None), \
        "You must provide roi OR subvol_bounds-zyx (but not both)."

    # Figure out what blocks ('substacks') we're copying
    if subvol_bounds_zyx:
        assert False, "User beware: The subvol_bounds_zyx option hasn't been tested yet. " \
                      "Now that you've been warned, comment out this assertion and give it a try. "\
                      "(It *should* work...)"

        assert len(subvol_bounds_zyx) == 2, "Invalid value for subvol_bounds_zyx"
        assert list(map(len, subvol_bounds_zyx)) == [3,3], "Invalid value for subvol_bounds_zyx"

        subvol_bounds_zyx = np.array(subvol_bounds_zyx)
        subvol_shape = subvol_bounds_zyx[1] - subvol_bounds_zyx[0]
        np.array(subvol_bounds_zyx) / transfer_cube_width_px
        assert (subvol_shape % transfer_cube_width_px).all(), \
            "subvolume must be divisible by the transfer_cube_width_px"
        
        blocks_zyx = []
        transfer_block_indexes = np.ndindex( *(subvol_shape / transfer_cube_width_px) )
        for tbi in transfer_block_indexes:
            start_zyx = tbi*transfer_cube_width_px + subvol_bounds_zyx[0]
            blocks_zyx.append( SubstackZYX(transfer_cube_width_px, *start_zyx) )        
    elif roi is not None:
        if isinstance(roi, str):
            roi_details = str_to_details( roi, default=source_details )
        else:
            roi_details = InstanceDetails(*roi)
        roi_node = DVIDNodeService(roi_details.host, roi_details.uuid)
        blocks_zyx = roi_node.get_roi_partition(roi_details.instance, transfer_cube_width_px/DVID_BLOCK_WIDTH)[0]
    else:
        assert False

    # Fetch/write the blocks one at a time
    # TODO: We could speed this up if we used a threadpool...
    logger.debug( "Beginning Transfer of {} blocks ({} px each)".format( len(blocks_zyx), transfer_cube_width_px ) )
    for block_index, block_zyx in enumerate(blocks_zyx, start=1):
        start_zyxc = np.array(tuple(block_zyx[1:]) + (0,)) # skip item 0 ('size'), append channel
        stop_zyxc = start_zyxc + transfer_cube_width_px
        stop_zyxc[-1] = 1

        logger.debug("Fetching block: {} ({}/{})".format(start_zyxc[:-1], block_index, len(blocks_zyx)) )
        src_block_data = src_accessor.get_ndarray( start_zyxc, stop_zyxc )
        
        logger.debug("Writing block:  {} ({}/{})".format(start_zyxc[:-1], block_index, len(blocks_zyx)) )
        dest_accessor.post_ndarray( start_zyxc, stop_zyxc, new_data=src_block_data )
        
    logger.debug("DONE.")

# 需要导入模块: from libdvid import DVIDNodeService [as 别名]
# 或者: from libdvid.DVIDNodeService import get_roi_partition [as 别名]

#.........这里部分代码省略.........
#             rollback_pred = True

        for iternum in range(0, num_iters):
            # it might make sense to randomly map partitions for selection
            # in case something pathological is happening -- if original partitioner
            # is randomish than this should be fine
            def subset_part(roi):
                s_id, data = roi
                if (s_id % num_iters) == iternum:
                    return True
                return False
            
            # should preserve partitioner
            distsubvolumes_part = distsubvolumes.filter(subset_part)

            # get grayscale chunks with specified overlap
            gray_chunks = self.sparkdvid_context.map_grayscale8(distsubvolumes_part,
                    self.config_data["dvid-info"]["grayscale"])


            # convert grayscale to compressed segmentation, maintain partitioner
            # save max id as well in substack info
            pred_checkpoint_dir = checkpoint_dir
            if checkpoint_dir != "":
                pred_checkpoint_dir = checkpoint_dir + "/prediter-" + str(iternum)

            # disable prediction checkpointing if rolling back at the iteration level
            # as this will cause unnecessary jobs to execute.  In principle, the
            # prediction could be rolled back as well which would only add some
            # unnecessary overhead to the per iteration rollback
            if rollback_seg:
                pred_checkpoint_dir = ""

            # small hack since segmentor is unaware for current iteration
            # perhaps just declare the segment function to have an arbitrary number of parameters
            if type(segmentor) == Segmentor:
                seg_chunks = segmentor.segment(gray_chunks, pred_checkpoint_dir, rollback_pred)
            else:
                seg_chunks = segmentor.segment(gray_chunks)

            # retrieve previously computed RDD or save current RDD
            if checkpoint_dir != "":
                seg_chunks = self.sparkdvid_context.checkpointRDD(seg_chunks, 
                        checkpoint_dir + "/segiter-" + str(iternum), rollback_seg)  

            # any forced persistence will result in costly
            # pickling, lz4 compressed numpy array should help
            seg_chunks.persist(StorageLevel.MEMORY_AND_DISK_SER)

            seg_chunks_list.append(seg_chunks)

        seg_chunks = seg_chunks_list[0]

        for iter1 in range(1, len(seg_chunks_list)):
            # ?? does this preserve the partitioner (yes, if num partitions is the same)
            # this could cause a serialization problems if there are a large number of iterations (>100)
            seg_chunks = seg_chunks.union(seg_chunks_list[iter1])
            
        # any forced persistence will result in costly
        # pickling, lz4 compressed numpy array should help
        seg_chunks.persist(StorageLevel.MEMORY_AND_DISK_SER)

        # stitch the segmentation chunks
        # (preserves initial partitioning)
        mapped_seg_chunks = segmentor.stitch(seg_chunks)
        
        # no longer need seg chunks
        seg_chunks.unpersist()

        # coalesce to fewer partitions (!!TEMPORARY SINCE THERE ARE WRITE BANDWIDTH LIMITS TO DVID)
        mapped_seg_chunks = mapped_seg_chunks.coalesce(125)

        # write data to DVID
        self.sparkdvid_context.foreach_write_labels3d(self.config_data["dvid-info"]["segmentation-name"], mapped_seg_chunks, self.config_data["dvid-info"]["roi"], self.config_data["options"]["mutateseg"])
        self.logger.write_data("Wrote DVID labels") # write to logger after spark job

        if self.config_data["options"]["debug"]:
            # grab 256 cube from ROI 
            from libdvid import DVIDNodeService
            node_service = DVIDNodeService(str(self.config_data["dvid-info"]["dvid-server"]),
                                           str(self.config_data["dvid-info"]["uuid"]))
            
            substacks, packing_factor = node_service.get_roi_partition(str(self.config_data["dvid-info"]["roi"]),
                                                                       256/self.blocksize)

            label_volume = node_service.get_labels3D( str(self.config_data["dvid-info"]["segmentation-name"]), 
                                                      (256,256,256),
                                                      (substacks[0][0], substacks[0][1], substacks[0][2]),
                                                      compress=True )
             
            # retrieve string
            from DVIDSparkServices.sparkdvid.CompressedNumpyArray import CompressedNumpyArray
            vol_compressed = CompressedNumpyArray(label_volume)
           
            # dump checksum
            import hashlib
            md5 = hashlib.md5()
            md5.update(vol_compressed.serialized_data[0])
            checksum_text = md5.hexdigest()
            print "DEBUG: ", checksum_text