Python DVIDNodeService.get_json方法代码示例

# 需要导入模块: from libdvid import DVIDNodeService [as 别名]
# 或者: from libdvid.DVIDNodeService import get_json [as 别名]
    def execute(self):
        # imports here so that schema can be retrieved without installation
        from DVIDSparkServices.reconutils import Evaluate
        from libdvid import DVIDNodeService
        from pyspark import SparkContext
        from pyspark import StorageLevel
        import time
        import datetime
        import json

        node_service = DVIDNodeService(str(self.config_data["dvid-info"]["dvid-server"]),
                                       str(self.config_data["dvid-info"]["uuid"]))

        if "chunk-size" in self.config_data["options"]:
            self.chunksize = self.config_data["options"]["chunk-size"]

        #  grab ROI (no overlap and no neighbor checking)
        distrois = self.sparkdvid_context.parallelize_roi(self.config_data["dvid-info"]["roi"],
                self.chunksize)

        # map ROI to two label volumes (0 overlap)
        # this will be used for all volume and point overlaps
        # (preserves partitioner)
        # (key, (subvolume, seggt, seg2)
        lpairs = self.sparkdvid_context.map_labels64_pair(
                distrois, self.config_data["dvid-info"]["label-name"],
                self.config_data["dvid-info-comp"]["dvid-server"],
                self.config_data["dvid-info-comp"]["uuid"],
                self.config_data["dvid-info-comp"]["label-name"], self.config_data["dvid-info"]["roi"])
       
        def _split_disjoint_labels(label_pairs):
            """Helper function: map subvolumes so disconnected bodies are different labels.

            Function preserves partitioner.

            Args:
                label_pairs (rdd): RDD is of (subvolume id, data)
       
            Returns:
                Original RDD including mappings for gt and the test seg.
        
            """
            from DVIDSparkServices.reconutils.morpho import split_disconnected_bodies
            from DVIDSparkServices.sparkdvid.CompressedNumpyArray import CompressedNumpyArray
            
            subvolume, labelgtc, label2c = label_pairs

            # extract numpy arrays
            labelgt = labelgtc.deserialize()
            label2 = label2c.deserialize()

            # split bodies up
            labelgt_split, labelgt_map = split_disconnected_bodies(labelgt)
            label2_split, label2_map = split_disconnected_bodies(label2)
            
            # compress results
            return (subvolume, labelgt_map, label2_map,
                    CompressedNumpyArray(labelgt_split),
                    CompressedNumpyArray(label2_split))


        # split bodies that are merged outside of the subvolume
        # (preserves partitioner)
        # => (key, (subvolume, seggt-split, seg2-split, seggt-map, seg2-map))
        lpairs_split = lpairs.mapValues(_split_disjoint_labels)

        # evaluation tool (support RAND, VI, per body, graph, and
        # histogram stats over different sets of points)
        evaluator = Evaluate.Evaluate(self.config_data)

        ### VOLUMETRIC ANALYSIS ###

        # TODO: !! Grab number of intersecting disjoint faces
        # (might need +1 border) for split edit distance
        
        # grab volumetric body overlap ignoring boundaries as specified
        # and generate overlap stats for substack (compute local)
        # => (key, (subvolume, stats, seggt-split, seg2-split, seggt-map, seg2-map))
        # (preserve partitioner)
        lpairs_proc = evaluator.calcoverlap(lpairs_split, self.config_data["options"]["boundary-size"])
       
        point_data = {}
        ### POINT ANALYSIS ###
        for point_list_name in self.config_data["dvid-info"]["point-lists"]:
            # grab point list from DVID
            keyvalue = point_list_name.split('/')
            if len(keyvalue) != 2:
                raise Exception(str(point_list_name) + "point list key value not properly specified")

            # is this too large to broadcast?? -- default lz4 should help quite a bit
            # TODO: send only necessary data to each job through join might help
            point_data[keyvalue[1]] = node_service.get_json(str(keyvalue[0]),
                    str(keyvalue[1]))
            
            # Generate per substack and global stats for given points.
            # Querying will just be done on the local labels stored.
            # (preserve partitioner)
            lpairs_proc = evaluator.calcoverlap_pts(lpairs_proc, keyvalue[1], point_data[keyvalue[1]])

        # Extract stats by retrieving substacks and stats info and
#.........这里部分代码省略.........