Python RequestPool.clean方法代码示例

# 需要导入模块: from lazyflow.request import RequestPool [as 别名]
# 或者: from lazyflow.request.RequestPool import clean [as 别名]
    def execute(self, slot, subindex, roi, result):
        assert slot == self._ReorderedOutput
        pool = RequestPool()

        t_ind = 0
        for t in range(roi.start[0], roi.stop[0]):
            c_ind = 0
            for c in range(roi.start[-1], roi.stop[-1]):
                newroi = roi.copy()
                newroi.start[0] = t
                newroi.stop[0] = t+1
                newroi.start[-1] = c
                newroi.stop[-1] = c+1

                req = self._op.Output.get(newroi)
                resView = result[t_ind:t_ind+1, ..., c_ind:c_ind+1]
                req.writeInto(resView)

                pool.add(req)

                c_ind += 1

            t_ind += 1

        pool.wait()
        pool.clean()

# 需要导入模块: from lazyflow.request import RequestPool [as 别名]
# 或者: from lazyflow.request.RequestPool import clean [as 别名]
    def _label(self, roi, result):
        result = vigra.taggedView(result, axistags=self.Output.meta.axistags)
        # get the background values
        bg = self.Background[...].wait()
        bg = vigra.taggedView(bg, axistags=self.Background.meta.axistags)
        bg = bg.withAxes(*'ct')
        assert np.all(self.Background.meta.shape[3:] ==
                      self.Input.meta.shape[3:]),\
            "Shape of background values incompatible to shape of Input"

        # do labeling in parallel over channels and time slices
        pool = RequestPool()

        start = np.asarray(roi.start, dtype=np.int)
        stop = np.asarray(roi.stop, dtype=np.int)
        for ti, t in enumerate(range(roi.start[4], roi.stop[4])):
            start[4], stop[4] = t, t+1
            for ci, c in enumerate(range(roi.start[3], roi.stop[3])):
                start[3], stop[3] = c, c+1
                newRoi = SubRegion(self.Output,
                                   start=tuple(start), stop=tuple(stop))
                resView = result[..., ci, ti].withAxes(*'xyz')
                req = Request(partial(self._label3d, newRoi,
                                      bg[c, t], resView))
                pool.add(req)

        logger.debug(
            "{}: Computing connected components for ROI {} ...".format(
                self.name, roi))
        pool.wait()
        pool.clean()
        logger.debug("{}: Connected components computed.".format(
            self.name))

# 需要导入模块: from lazyflow.request import RequestPool [as 别名]
# 或者: from lazyflow.request.RequestPool import clean [as 别名]
    def execute(self, slot, subindex, roi, result):
        with self._lock:
            if self.cache is None:
                fullBlockShape = numpy.array([self.blockShape.value for i in self.Input.meta.shape])
                fun = self.inputs["Function"].value
                #data = self.inputs["Input"][:].wait()
                #split up requests into blocks
                shape = self.Input.meta.shape
                numBlocks = numpy.ceil(shape/(1.0*fullBlockShape)).astype("int")
                blockCache = numpy.ndarray(shape = numpy.prod(numBlocks), dtype=self.Output.meta.dtype)
                pool = RequestPool()
                #blocks holds the different roi keys for each of the blocks
                blocks = itertools.product(*[range(i) for i in numBlocks])
                blockKeys = []
                for b in blocks:
                    start = b * fullBlockShape
                    stop = b * fullBlockShape + fullBlockShape
                    stop = numpy.min(numpy.vstack((stop, shape)), axis=0)
                    blockKey = roiToSlice(start, stop)
                    blockKeys.append(blockKey)
                
                def predict_block(i):
                    data = self.Input[blockKeys[i]].wait()
                    blockCache[i] = fun(data)
                    
                for i,f in enumerate(blockCache):
                    req = pool.request(partial(predict_block,i))

                pool.wait()
                pool.clean()

                self.cache = [fun(blockCache)]
            return self.cache

# 需要导入模块: from lazyflow.request import RequestPool [as 别名]
# 或者: from lazyflow.request.RequestPool import clean [as 别名]
    def execute(self, slot, subindex, rroi, result):
        key = roiToSlice(rroi.start,rroi.stop)

        cnt = 0
        written = 0
        start, stop = roi.sliceToRoi(key, self.outputs["Output"].meta.shape)
        assert (stop<=self.outputs["Output"].meta.shape).all()
        #axisindex = self.inputs["AxisIndex"].value
        flag = self.inputs["AxisFlag"].value
        axisindex = self.outputs["Output"].meta.axistags.index(flag)
        #ugly-ugly-ugly
        oldkey = list(key)
        oldkey.pop(axisindex)
        
        #print "STACKER: ", flag, axisindex
        #print "requesting an outslot from stacker:", key, result.shape
        #print "input slots total: ", len(self.inputs['Images'])
        requests = []
        
        pool = RequestPool()

        for i, inSlot in enumerate(self.inputs['Images']):
            req = None
            inTagKeys = [ax.key for ax in inSlot.meta.axistags]
            if flag in inTagKeys:
                slices = inSlot.meta.shape[axisindex]
                if cnt + slices >= start[axisindex] and start[axisindex]-cnt<slices and start[axisindex]+written<stop[axisindex]:
                    begin = 0
                    if cnt < start[axisindex]:
                        begin = start[axisindex] - cnt
                    end = slices
                    if cnt + end > stop[axisindex]:
                        end -= cnt + end - stop[axisindex]
                    key_ = copy.copy(oldkey)
                    key_.insert(axisindex, slice(begin, end, None))
                    reskey = [slice(None, None, None) for x in range(len(result.shape))]
                    reskey[axisindex] = slice(written, written+end-begin, None)

                    req = inSlot[tuple(key_)].writeInto(result[tuple(reskey)])
                    written += end - begin
                cnt += slices
            else:
                if cnt>=start[axisindex] and start[axisindex] + written < stop[axisindex]:
                    #print "key: ", key, "reskey: ", reskey, "oldkey: ", oldkey
                    #print "result: ", result.shape, "inslot:", inSlot.meta.shape
                    reskey = [slice(None, None, None) for s in oldkey]
                    reskey.insert(axisindex, written)
                    destArea = result[tuple(reskey)]
                    req = inSlot[tuple(oldkey)].writeInto(destArea)
                    written += 1
                cnt += 1

            if req is not None:
                pool.add(req)

        pool.wait()
        pool.clean()

# 需要导入模块: from lazyflow.request import RequestPool [as 别名]
# 或者: from lazyflow.request.RequestPool import clean [as 别名]
    def execute(self, slot, subindex, roi, result):
        t1 = time.time()
        key = roi.toSlice()
        nlabels=self.inputs["LabelsCount"].value

        traceLogger.debug("OpPredictRandomForest: Requesting classifier. roi={}".format(roi))
        forests=self.inputs["Classifier"][:].wait()

        if forests is None or any(x is None for x in forests):
            # Training operator may return 'None' if there was no data to train with
            return numpy.zeros(numpy.subtract(roi.stop, roi.start), dtype=numpy.float32)[...]

        traceLogger.debug("OpPredictRandomForest: Got classifier")
        #assert RF.labelCount() == nlabels, "ERROR: OpPredictRandomForest, labelCount differs from true labelCount! %r vs. %r" % (RF.labelCount(), nlabels)

        newKey = key[:-1]
        newKey += (slice(0,self.inputs["Image"].meta.shape[-1],None),)

        res = self.inputs["Image"][newKey].wait()

        shape=res.shape
        prod = numpy.prod(shape[:-1])
        res.shape = (prod, shape[-1])
        features=res

        predictions = [0]*len(forests)

        def predict_forest(number):
            predictions[number] = forests[number].predictProbabilities(numpy.asarray(features, dtype=numpy.float32))

        t2 = time.time()

        # predict the data with all the forests in parallel
        pool = RequestPool()

        for i,f in enumerate(forests):
            req = pool.request(partial(predict_forest, i))

        pool.wait()
        pool.clean()

        prediction=numpy.dstack(predictions)
        prediction = numpy.average(prediction, axis=2)
        prediction.shape =  shape[:-1] + (forests[0].labelCount(),)
        #prediction = prediction.reshape(*(shape[:-1] + (forests[0].labelCount(),)))

        # If our LabelsCount is higher than the number of labels in the training set,
        # then our results aren't really valid.  FIXME !!!
        # Duplicate the last label's predictions
        for c in range(result.shape[-1]):
            result[...,c] = prediction[...,min(c+key[-1].start, prediction.shape[-1]-1)]

        t3 = time.time()

        self.logger.debug("predict roi=%r took %fseconds, actual RF time was %fs, feature time was %fs" % (key, t3-t1, t3-t2, t2-t1))
        
        return result

# 需要导入模块: from lazyflow.request import RequestPool [as 别名]
# 或者: from lazyflow.request.RequestPool import clean [as 别名]
    def execute(self, slot, subindex, roi, result):

        featList = []
        labelsList = []

        for i in range(len(self.Labels)):
            feats = self.Features[i]([]).wait()

            # TODO: we should be able to use self.Labels[i].value,
            # but the current implementation of Slot.value() does not
            # do the right thing.
            labels = self.Labels[i]([]).wait()

            featstmp, labelstmp = make_feature_array(feats, labels)
            featList.append(featstmp)
            labelsList.append(labelstmp)

        featMatrix = _concatenate(featList, axis=0)
        labelsMatrix = _concatenate(labelsList, axis=0)
        print "training on matrix:", featMatrix.shape, featMatrix.dtype

        if len(featMatrix) == 0 or len(labelsMatrix) == 0:
            result[:] = None
            return
        oob = [0] * self.ForestCount.value
        try:
            # Ensure there are no NaNs in the feature matrix
            # TODO: There should probably be a better way to fix this...
            featMatrix = numpy.asarray(featMatrix, dtype=numpy.float32)
            nanFeatMatrix = numpy.isnan(featMatrix)
            if nanFeatMatrix.any():
                warnings.warn("Feature matrix has NaN values!  Replacing with 0.0...")
                featMatrix[numpy.where(nanFeatMatrix)] = 0.0
            # train and store forests in parallel
            pool = RequestPool()
            for i in range(self.ForestCount.value):
                def train_and_store(number):
                    result[number] = vigra.learning.RandomForest(self._tree_count)
                    oob[number] = result[number].learnRF(featMatrix, numpy.asarray(labelsMatrix, dtype=numpy.uint32))
                    print "intermediate oob:", oob[number]
                req = Request( partial(train_and_store, i) )
                pool.add( req )
            pool.wait()
            pool.clean()
        except:
            print ("couldn't learn classifier")
            raise
        oob_total = numpy.mean(oob)
        print "training finished, out of bag error:", oob_total
        return result

# 需要导入模块: from lazyflow.request import RequestPool [as 别名]
# 或者: from lazyflow.request.RequestPool import clean [as 别名]
    def predict(cls, X, method='classic'):
        """
        predict if the histograms in X correspond to missing regions
        do this for subsets of X in parallel
        """

        if cls._manager is None:
            cls._manager = SVMManager()

        # svm input has to be (nSamples, nFeatures) -> for us: (nSampels = len(X), nFeatures = # of histogrambins )
        X_reshaped = np.zeros((len(X), len(X[0])))
        for i in range(len(X)):
            X_reshaped[i, :] = X[i]

        n_bins = len(X[0])

        if method == 'classic' or not have_sklearn:
            logger.warning("no real svm used! -> PseudoSVC")
            svm = PseudoSVC()
        else:
            # load samples for histograms of labeled regions
            try:
                svm = cls._manager.get(n_bins)
            except SVMManager.NotTrainedError:
                # fail gracefully if not trained => responsibility of user!
                svm = PseudoSVC()

        y = np.zeros((len(X),)) * np.nan

        pool = RequestPool()

        # chunk up all samples from X into chunks that will be predicted in parallel
        chunk_size = 1000  # FIXME magic number??
        n_chunks = len(X)/chunk_size + (1 if len(X) % chunk_size > 0 else 0)

        s = [slice(k * chunk_size, min((k + 1) * chunk_size, len(X)))
             for k in range(n_chunks)]

        def partFun(i):
            y[s[i]] = svm.predict(X_reshaped[s[i], :])

        for i in range(n_chunks):
            req = Request(partial(partFun, i))
            pool.add(req)

        pool.wait()
        pool.clean()
        return np.asarray(y)

# 需要导入模块: from lazyflow.request import RequestPool [as 别名]
# 或者: from lazyflow.request.RequestPool import clean [as 别名]
    def execute(self, slot, subindex, roi, result):
        with self._lock:
            if self.cache is None:
                shape = self.Input.meta.shape
                # self.blockshape has None in the last dimension to indicate that it should not be
                # handled block-wise. None is replaced with the image shape in the respective axis.
                fullBlockShape = []
                for u, v in zip(self.blockShape.value, shape):
                    if u is not None:
                        fullBlockShape.append(u)
                    else:
                        fullBlockShape.append(v)
                fullBlockShape = numpy.array(fullBlockShape, dtype=numpy.float64)

                #data = self.inputs["Input"][:].wait()
                #split up requests into blocks

                numBlocks = numpy.ceil(shape / fullBlockShape).astype("int")
                blockCache = numpy.ndarray(shape = numpy.prod(numBlocks), dtype=self.Output.meta.dtype)
                pool = RequestPool()
                #blocks holds the different roi keys for each of the blocks
                blocks = itertools.product(*[list(range(i)) for i in numBlocks])
                blockKeys = []
                for b in blocks:
                    start = b * fullBlockShape
                    stop = b * fullBlockShape + fullBlockShape
                    stop = numpy.min(numpy.vstack((stop, shape)), axis=0)
                    blockKey = roiToSlice(start, stop)
                    blockKeys.append(blockKey)

                fun = self.inputs["Function"].value
                def predict_block(i):
                    data = self.Input[blockKeys[i]].wait()
                    blockCache[i] = fun(data)

                for i,f in enumerate(blockCache):
                    req = pool.request(partial(predict_block,i))

                pool.wait()
                pool.clean()

                self.cache = [fun(blockCache)]
            return self.cache

# 需要导入模块: from lazyflow.request import RequestPool [as 别名]
# 或者: from lazyflow.request.RequestPool import clean [as 别名]
    def execute(self, slot, subindex, roi, result):
        featMatrix = []
        labelsMatrix = []

        for i in range(len(self.Labels)):
            feats = self.Features[i]([]).wait()

            # TODO: we should be able to use self.Labels[i].value,
            # but the current implementation of Slot.value() does not
            # do the right thing.
            labels = self.Labels[i]([]).wait()

            for t in sorted(feats.keys()):
                featsMatrix_tmp = []
                labelsMatrix_tmp = []
                lab = labels[t].squeeze()
                index = numpy.nonzero(lab)
                labelsMatrix_tmp.append(lab[index])

                #check that all requested features are present
                for featname in config.selected_features:
                    for channel in feats[t]:
                        if not featname in channel.keys():
                            print "Feature", featname, "has not been computed in the previous step"
                            print "We only have the following features now:", channel.keys()
                            result[:] = None
                            return
                        else:
                            value = channel[featname]
                            ft = numpy.asarray(value.squeeze())
                            featsMatrix_tmp.append(ft[index])
                
                featMatrix.append(_concatenate(featsMatrix_tmp, axis=1))
                labelsMatrix.append(_concatenate(labelsMatrix_tmp, axis=1))

        featMatrix = _concatenate(featMatrix, axis=0)
        labelsMatrix = _concatenate(labelsMatrix, axis=0)
        print "training on matrix:", featMatrix.shape
        
        if len(featMatrix) == 0 or len(labelsMatrix) == 0:
            result[:] = None
            return
        oob = [0]*self.ForestCount.value
        try:
            # Ensure there are no NaNs in the feature matrix
            # TODO: There should probably be a better way to fix this...
            featMatrix = numpy.asarray(featMatrix, dtype=numpy.float32)
            nanFeatMatrix = numpy.isnan(featMatrix)
            if nanFeatMatrix.any():
                warnings.warn("Feature matrix has NaN values!  Replacing with 0.0...")
                featMatrix[numpy.where(nanFeatMatrix)] = 0.0
            # train and store forests in parallel
            pool = RequestPool()
            for i in range(self.ForestCount.value):
                def train_and_store(number):
                    result[number] = vigra.learning.RandomForest(self._tree_count)
                    oob[number] = result[number].learnRF(featMatrix, numpy.asarray(labelsMatrix, dtype=numpy.uint32))
                    print "intermediate oob:", oob[number]
                req = Request( partial(train_and_store, i) )
                pool.add( req )
            pool.wait()
            pool.clean()
        except:
            print ("couldn't learn classifier")
            raise
        oob_total = numpy.mean(oob)
        print "training finished, out of bag error:", oob_total
        return result

# 需要导入模块: from lazyflow.request import RequestPool [as 别名]
# 或者: from lazyflow.request.RequestPool import clean [as 别名]

#.........这里部分代码省略.........
                traceLogger.debug("Requests fired")
                

                #Fixme: Maybe later request only part of the region?

                #image=self.inputs["Images"][i][:].wait()
                for ir, req in enumerate(reqlistlabels):
                    
                    labblock = req.wait()
                    
                    image = reqlistfeat[ir].wait()
                    labbgblock = reqlistbg[ir].wait()
                    labblock = labblock.reshape((image.shape[:-1]))
                    image = image.reshape((-1, image.shape[-1]))
                    labbgindices = np.where(labbgblock == 2)            
                    labbgindices = np.ravel_multi_index(labbgindices, labbgblock.shape)
                    
                    newDot, mapping, tags = \
                    self._svr.prepareDataRefactored(labblock, labbgindices)
                    #self._svr.prepareData(labblock, smooth = True)

                    labels   = newDot[mapping]
                    features = image[mapping]

                    featMatrix.append(features)
                    labelsMatrix.append(labels)
                    tagList.append(tags)
                
                progress = progress_outer[0]

                traceLogger.debug("Requests processed")


        self.progressSignal(80 / numImages)
        if len(featMatrix) == 0 or len(labelsMatrix) == 0:
            result[:] = None

        else:
            posTags = [tag[0] for tag in tagList]
            negTags = [tag[1] for tag in tagList]
            numPosTags = np.sum(posTags)
            numTags = np.sum(posTags) + np.sum(negTags)
            fullFeatMatrix = np.ndarray((numTags, self.Images[0].meta.shape[-1]), dtype = np.float64)
            fullLabelsMatrix = np.ndarray((numTags), dtype = np.float64)
            fullFeatMatrix[:] = np.NAN
            fullLabelsMatrix[:] = np.NAN
            currPosCount = 0
            currNegCount = numPosTags
            for i, posCount in enumerate(posTags):
                fullFeatMatrix[currPosCount:currPosCount + posTags[i],:] = featMatrix[i][:posCount,:]
                fullLabelsMatrix[currPosCount:currPosCount + posTags[i]] = labelsMatrix[i][:posCount]
                fullFeatMatrix[currNegCount:currNegCount + negTags[i],:] = featMatrix[i][posCount:,:]
                fullLabelsMatrix[currNegCount:currNegCount + negTags[i]] = labelsMatrix[i][posCount:]
                currPosCount += posTags[i]
                currNegCount += negTags[i]


            assert(not np.isnan(np.sum(fullFeatMatrix)))

            fullTags = [np.sum(posTags), np.sum(negTags)]
            #pool = RequestPool()

            maxima = np.max(fullFeatMatrix, axis=0)
            minima = np.min(fullFeatMatrix, axis=0)
            normalizationFactors = (minima,maxima)
            



            boxConstraintList = []
            boxConstraints = None
            if self.BoxConstraintRois.ready() and self.BoxConstraintValues.ready():
                for i, slot in enumerate(zip(self.BoxConstraintRois,self.BoxConstraintValues)):
                    for constr, val in zip(slot[0].value, slot[1].value):
                        boxConstraintList.append((i, constr, val))
                if len(boxConstraintList) > 0:
                    boxConstraints = self.constructBoxConstraints(boxConstraintList)

            params = self._svr.get_params() 
            try:
                pool = RequestPool()
                def train_and_store(i):
                    result[i] = SVR(minmax = normalizationFactors, **params)
                    result[i].fitPrepared(fullFeatMatrix, fullLabelsMatrix, tags = fullTags, boxConstraints = boxConstraints, numRegressors
                         = self.numRegressors, trainAll = False)
                for i in range(self.numRegressors):
                    req = pool.request(partial(train_and_store, i))
                
                pool.wait()
                pool.clean()
            
            except:
                logger.error("ERROR: could not learn regressor")
                logger.error("fullFeatMatrix shape = {}, dtype = {}".format(fullFeatMatrix.shape, fullFeatMatrix.dtype) )
                logger.error("fullLabelsMatrix shape = {}, dtype = {}".format(fullLabelsMatrix.shape, fullLabelsMatrix.dtype) )
                raise
            finally:
                self.progressSignal(100) 

        return result

# 需要导入模块: from lazyflow.request import RequestPool [as 别名]
# 或者: from lazyflow.request.RequestPool import clean [as 别名]
    def execute(self, slot, subindex, roi, result):
        assert slot in [self.Predictions,
                        self.Probabilities,
                        self.CachedProbabilities,
                        self.ProbabilityChannels,
                        self.BadObjects]

        times = roi._l
        if len(times) == 0:
            # we assume that 0-length requests are requesting everything
            times = range(self.Predictions.meta.shape[0])

        if slot is self.CachedProbabilities:
            return {t: self.prob_cache[t] for t in times if t in self.prob_cache}

        forests=self.inputs["Classifier"][:].wait()
        if forests is None or forests[0] is None:
            # this happens if there was no data to train with
            return dict((t, numpy.array([])) for t in times)

        feats = {}
        prob_predictions = {}

        selected = self.SelectedFeatures([]).wait()

        # FIXME: self.prob_cache is shared, so we need to block.
        # However, this makes prediction single-threaded.
        self.lock.acquire()
        try:
            for t in times:
                if t in self.prob_cache:
                    continue

                tmpfeats = self.Features([t]).wait()
                ftmatrix, _, col_names = make_feature_array(tmpfeats, selected)
                rows, cols = replace_missing(ftmatrix)
                self.bad_objects[t] = numpy.zeros((ftmatrix.shape[0],))
                self.bad_objects[t][rows] = 1
                feats[t] = ftmatrix
                prob_predictions[t] = [0] * len(forests)

            def predict_forest(_t, forest_index):
                # Note: We can't use RandomForest.predictLabels() here because we're training in parallel,
                #        and we have to average the PROBABILITIES from all forests.
                #       Averaging the label predictions from each forest is NOT equivalent.
                #       For details please see wikipedia:
                #       http://en.wikipedia.org/wiki/Electoral_College_%28United_States%29#Irrelevancy_of_national_popular_vote
                #       (^-^)
                prob_predictions[_t][forest_index] = forests[forest_index].predictProbabilities(feats[_t].astype(numpy.float32))

            # predict the data with all the forests in parallel
            pool = RequestPool()
            for t in times:
                if t in self.prob_cache:
                    continue
                for i, f in enumerate(forests):
                    req = Request( partial(predict_forest, t, i) )
                    pool.add(req)

            pool.wait()
            pool.clean()

            for t in times:
                if t not in self.prob_cache:
                    # prob_predictions is a dict-of-lists-of-arrays, indexed as follows:
                    # prob_predictions[t][forest_index][object_index, class_index]

                    # Stack the forests together and average them.
                    stacked_predictions = numpy.array( prob_predictions[t] )
                    averaged_predictions = numpy.average( stacked_predictions, axis=0 )
                    assert averaged_predictions.shape[0] == len(feats[t])
                    self.prob_cache[t] = averaged_predictions

                    self.prob_cache[t][0] = 0 # Background probability is always zero


            if slot == self.Probabilities:
                return { t : self.prob_cache[t] for t in times }
            elif slot == self.Predictions:
                # FIXME: Support SegmentationThreshold again...
                labels = dict()
                for t in times:
                    prob_sum = numpy.sum(self.prob_cache[t], axis=1)
                    labels[t] = 1 + numpy.argmax(self.prob_cache[t], axis=1)
                    labels[t][0] = 0 # Background gets the zero label
                
                return labels

            elif slot == self.ProbabilityChannels:
                try:
                    prob_single_channel = {t: self.prob_cache[t][:, subindex[0]]
                                           for t in times}
                except:
                    # no probabilities available for this class; return zeros
                    prob_single_channel = {t: numpy.zeros((self.prob_cache[t].shape[0], 1))
                                           for t in times}
                return prob_single_channel

            elif slot == self.BadObjects:
                return { t : self.bad_objects[t] for t in times }
#.........这里部分代码省略.........

# 需要导入模块: from lazyflow.request import RequestPool [as 别名]
# 或者: from lazyflow.request.RequestPool import clean [as 别名]
    def execute(self, slot, subindex, roi, result):
        progress = 0
        self.progressSignal(progress)
        numImages = len(self.Images)

        key = roi.toSlice()
        featMatrix=[]
        labelsMatrix=[]
        for i,labels in enumerate(self.inputs["Labels"]):
            if labels.meta.shape is not None:
                #labels=labels[:].wait()
                blocks = self.inputs["nonzeroLabelBlocks"][i][0].wait()

                progress += 10/numImages
                self.progressSignal(progress)

                reqlistlabels = []
                reqlistfeat = []
                traceLogger.debug("Sending requests for {} non-zero blocks (labels and data)".format( len(blocks[0])) )
                for b in blocks[0]:

                    request = labels[b]
                    featurekey = list(b)
                    featurekey[-1] = slice(None, None, None)
                    request2 = self.inputs["Images"][i][featurekey]

                    reqlistlabels.append(request)
                    reqlistfeat.append(request2)

                traceLogger.debug("Requests prepared")

                numLabelBlocks = len(reqlistlabels)
                progress_outer = [progress] # Store in list for closure access
                if numLabelBlocks > 0:
                    progressInc = (80-10)/numLabelBlocks/numImages

                def progressNotify(req):
                    # Note: If we wanted perfect progress reporting, we could use lock here
                    #       to protect the progress from being incremented simultaneously.
                    #       But that would slow things down and imperfect reporting is okay for our purposes.
                    progress_outer[0] += progressInc/2
                    self.progressSignal(progress_outer[0])

                for ir, req in enumerate(reqlistfeat):
                    image = req.notify_finished(progressNotify)

                for ir, req in enumerate(reqlistlabels):
                    labblock = req.notify_finished(progressNotify)

                traceLogger.debug("Requests fired")

                for ir, req in enumerate(reqlistlabels):
                    traceLogger.debug("Waiting for a label block...")
                    labblock = req.wait()

                    traceLogger.debug("Waiting for an image block...")
                    image = reqlistfeat[ir].wait()

                    indexes=numpy.nonzero(labblock[...,0].view(numpy.ndarray))
                    features=image[indexes]
                    labbla=labblock[indexes]

                    featMatrix.append(features)
                    labelsMatrix.append(labbla)

                progress = progress_outer[0]

                traceLogger.debug("Requests processed")

        self.progressSignal(80/numImages)

        if len(featMatrix) == 0 or len(labelsMatrix) == 0:
            # If there was no actual data for the random forest to train with, we return None
            result[:] = None
        else:
            featMatrix=numpy.concatenate(featMatrix,axis=0)
            labelsMatrix=numpy.concatenate(labelsMatrix,axis=0)
            maxLabel = self.inputs["MaxLabel"].value
            labelList = range(1, maxLabel+1) if maxLabel > 0 else list()

            try:
                logger.debug("Learning with Vigra...")
                # train and store self._forest_count forests in parallel
                pool = RequestPool()

                for i in range(self._forest_count):
                    def train_and_store(number):
                        result[number] = vigra.learning.RandomForest(self._tree_count, labels=labelList)
                        result[number].learnRF( numpy.asarray(featMatrix, dtype=numpy.float32),
                                                numpy.asarray(labelsMatrix, dtype=numpy.uint32))
                    req = pool.request(partial(train_and_store, i))

                pool.wait()
                pool.clean()

                logger.debug("Vigra finished")
            except:
                logger.error( "ERROR: could not learn classifier" )
                logger.error( "featMatrix shape={}, max={}, dtype={}".format(featMatrix.shape, featMatrix.max(), featMatrix.dtype) )
                logger.error( "labelsMatrix shape={}, max={}, dtype={}".format(labelsMatrix.shape, labelsMatrix.max(), labelsMatrix.dtype ) )
#.........这里部分代码省略.........

# 需要导入模块: from lazyflow.request import RequestPool [as 别名]
# 或者: from lazyflow.request.RequestPool import clean [as 别名]
    def execute(self, slot, subindex, slot_roi, target):
        assert slot == self.Features or slot == self.Output
        if slot == self.Features:
            feature_slice = roiToSlice(slot_roi.start, slot_roi.stop)
            index = subindex[0]
            feature_slice = list(feature_slice)

            # Translate channel slice of this feature to the channel slice of the output slot.
            output_channel_offset = self.featureOutputChannels[index][0]
            feature_slice[1] = slice(
                output_channel_offset + feature_slice[1].start, output_channel_offset + feature_slice[1].stop
            )
            slot_roi = SubRegion(self.Output, pslice=feature_slice)

            # Get output slot region for this channel
            return self.execute(self.Output, (), slot_roi, target)
        elif slot == self.Output:
            # Correlation of variable 'families' representing reference frames:
            #  ______________________________
            # | input/output frame           |  input/output shape given by slots
            # |  _________________________   |
            # | | smooth frame            |  |  pre-smoothing op needs halo around filter roi
            # | |  ____________________   |  |
            # | | |filter frame        |  |  |  filter needs halo around target roi
            # | | |  _______________   |  |  |
            # | | | | target frame  |  |  |  |  target is given by output_roi

            # note: The 'full_' variable prefix refers to the full 5D shape (tczyx), without 'full_' variables mostly
            #       refer to the 3D space subregion (zyx)

            full_output_slice = slot_roi.toSlice()

            logger.debug(f"OpPixelFeaturesPresmoothed: request {slot_roi.pprint()}")

            assert (slot_roi.stop <= self.Output.meta.shape).all()

            full_output_shape = self.Output.meta.shape
            full_output_start, full_output_stop = sliceToRoi(full_output_slice, full_output_shape)
            assert len(full_output_shape) == 5
            if all(self.ComputeIn2d.value):  # todo: check for this particular slice
                axes2enlarge = (0, 1, 1)
            else:
                axes2enlarge = (1, 1, 1)

            output_shape = full_output_shape[2:]
            output_start = full_output_start[2:]
            output_stop = full_output_stop[2:]

            axistags = self.Output.meta.axistags
            target = target.view(vigra.VigraArray)
            target.axistags = copy.copy(axistags)

            # filter roi in input frame
            # sigma = 0.7, because the features receive a pre-smoothed array and don't need much of a neighborhood
            input_filter_start, input_filter_stop = roi.enlargeRoiForHalo(
                output_start, output_stop, output_shape, 0.7, self.WINDOW_SIZE, enlarge_axes=axes2enlarge
            )

            # smooth roi in input frame
            input_smooth_start, input_smooth_stop = roi.enlargeRoiForHalo(
                input_filter_start,
                input_filter_stop,
                output_shape,
                self.max_sigma,
                self.WINDOW_SIZE,
                enlarge_axes=axes2enlarge,
            )

            # target roi in filter frame
            filter_target_start = roi.TinyVector(output_start - input_filter_start)
            filter_target_stop = roi.TinyVector(output_stop - input_filter_start)

            # filter roi in smooth frame
            smooth_filter_start = roi.TinyVector(input_filter_start - input_smooth_start)
            smooth_filter_stop = roi.TinyVector(input_filter_stop - input_smooth_start)

            filter_target_slice = roi.roiToSlice(filter_target_start, filter_target_stop)
            input_smooth_slice = roi.roiToSlice(input_smooth_start, input_smooth_stop)

            # pre-smooth for all requested time slices and all channels
            full_input_smooth_slice = (full_output_slice[0], slice(None), *input_smooth_slice)
            req = self.Input[full_input_smooth_slice]
            source = req.wait()
            req.clean()
            req.destination = None
            if source.dtype != numpy.float32:
                sourceF = source.astype(numpy.float32)
                try:
                    source.resize((1,), refcheck=False)
                except Exception:
                    pass
                del source
                source = sourceF

            sourceV = source.view(vigra.VigraArray)
            sourceV.axistags = copy.copy(self.Input.meta.axistags)

            dimCol = len(self.scales)
            dimRow = self.matrix.shape[0]

#.........这里部分代码省略.........

# 需要导入模块: from lazyflow.request import RequestPool [as 别名]
# 或者: from lazyflow.request.RequestPool import clean [as 别名]
    def execute(self, slot, subindex, roi, result):
        progress = 0
        self.progressSignal(progress)
        numImages = len(self.Images)

        featMatrix=[]
        labelsMatrix=[]
        tagsMatrix = []

        result[0] = SVR(self.UnderMult.value, self.OverMult.value, limitDensity = True, **self.SelectedOption.value)
        for i,labels in enumerate(self.inputs["Labels"]):
            if labels.meta.shape is not None:
                #labels=labels[:].wait()
                blocks = self.inputs["nonzeroLabelBlocks"][i][0].wait()

                progress += 10/numImages
                self.progressSignal(progress)

                reqlistlabels = []
                reqlistfeat = []
                traceLogger.debug("Sending requests for {} non-zero blocks (labels and data)".format( len(blocks[0])) )
                for b in blocks[0]:

                    request = labels[b]
                    featurekey = list(b)
                    featurekey[-1] = slice(None, None, None)
                    request2 = self.inputs["Images"][i][featurekey]

                    reqlistlabels.append(request)
                    reqlistfeat.append(request2)

                traceLogger.debug("Requests prepared")

                numLabelBlocks = len(reqlistlabels)
                progress_outer = [progress] # Store in list for closure access
                if numLabelBlocks > 0:
                    progressInc = (80-10)/numLabelBlocks/numImages

                def progressNotify(req):
                    # Note: If we wanted perfect progress reporting, we could use lock here
                    #       to protect the progress from being incremented simultaneously.
                    #       But that would slow things down and imperfect reporting is okay for our purposes.
                    progress_outer[0] += progressInc/2
                    self.progressSignal(progress_outer[0])

                for ir, req in enumerate(reqlistfeat):
                    image = req.notify_finished(progressNotify)

                for ir, req in enumerate(reqlistlabels):
                    labblock = req.notify_finished(progressNotify)

                traceLogger.debug("Requests fired")

                for ir, req in enumerate(reqlistlabels):
                    traceLogger.debug("Waiting for a label block...")
                    labblock = req.wait()

                    traceLogger.debug("Waiting for an image block...")
                    image = reqlistfeat[ir].wait()

                    newImg, newDot, mapping, tags = \
                    result[0].prepareData(image, labblock, sigma = self.Sigma.value, smooth = True, normalize = False)

                    features = newImg[mapping]
                    labbla = newDot[mapping]

                    #indexes=np.nonzero(labblock[...,0].view(np.ndarray))
                    #features=image[indexes]
                    #labbla=labblock[indexes]

                    featMatrix.append(features)
                    labelsMatrix.append(labbla)
                    tagsMatrix.append(tags)

                progress = progress_outer[0]

                traceLogger.debug("Requests processed")

        self.progressSignal(80/numImages)

        if len(featMatrix) == 0 or len(labelsMatrix) == 0:
            # If there was no actual data for the random forest to train with, we return None
            result[:] = None
        else:
            featMatrix=np.concatenate(featMatrix,axis=0)
            labelsMatrix=np.concatenate(labelsMatrix,axis=0)
            tagsMatrix=np.concatenate(tagsMatrix,axis=0)

            try:
                logger.debug("Learning with Vigra...")

                pool = RequestPool()

                #result[0].fitPrepared(featMatrix, labelsMatrix, tagsMatrix, self.Epsilon.value)
                req = pool.request(partial(result[0].fitPrepared, featMatrix, labelsMatrix, tagsMatrix, self.Epsilon.value))
                pool.wait()
                pool.clean()

                logger.debug("Vigra finished")
            except:
#.........这里部分代码省略.........

# 需要导入模块: from lazyflow.request import RequestPool [as 别名]
# 或者: from lazyflow.request.RequestPool import clean [as 别名]
    def _execute_graphcut(self, roi, result):
        for i in (0, 4):
            assert roi.stop[i] - roi.start[i] == 1,\
                "Invalid roi for graph-cut: {}".format(str(roi))
        t = roi.start[0]
        c = roi.start[4]

        margin = self.Margin.value
        beta = self.Beta.value
        MAXBOXSIZE = 10000000  # FIXME justification??

        ## request the bounding box coordinates ##
        # the trailing index brackets give us the dictionary (instead of an
        # array of size 1)
        feats = self.BoundingBoxes.get(roi).wait()
        mins = feats["Coord<Minimum>"]
        maxs = feats["Coord<Maximum>"]
        nobj = mins.shape[0]
        # these are indices, so they should have an index datatype
        mins = mins.astype(np.uint32)
        maxs = maxs.astype(np.uint32)

        ## request the prediction image ##
        pred = self.Prediction.get(roi).wait()
        pred = vigra.taggedView(pred, axistags=self.Prediction.meta.axistags)
        pred = pred.withAxes(*'xyz')

        ## request the connected components image ##
        cc = self.LabelImage.get(roi).wait()
        cc = vigra.taggedView(cc, axistags=self.LabelImage.meta.axistags)
        cc = cc.withAxes(*'xyz')

        # provide xyz view for the output (just need 8bit for segmentation
        resultXYZ = vigra.taggedView(np.zeros(cc.shape, dtype=np.uint8),
                                     axistags='xyz')

        def processSingleObject(i):
            logger.debug("processing object {}".format(i))
            # maxs are inclusive, so we need to add 1
            xmin = max(mins[i][0]-margin[0], 0)
            ymin = max(mins[i][1]-margin[1], 0)
            zmin = max(mins[i][2]-margin[2], 0)
            xmax = min(maxs[i][0]+margin[0]+1, cc.shape[0])
            ymax = min(maxs[i][1]+margin[1]+1, cc.shape[1])
            zmax = min(maxs[i][2]+margin[2]+1, cc.shape[2])
            ccbox = cc[xmin:xmax, ymin:ymax, zmin:zmax]
            resbox = resultXYZ[xmin:xmax, ymin:ymax, zmin:zmax]

            nVoxels = ccbox.size
            if nVoxels > MAXBOXSIZE:
                #problem too large to run graph cut, assign to seed
                logger.warn("Object {} too large for graph cut.".format(i))
                resbox[ccbox == i] = 1
                return

            probbox = pred[xmin:xmax, ymin:ymax, zmin:zmax]
            gcsegm = segmentGC(probbox, beta)
            gcsegm = vigra.taggedView(gcsegm, axistags='xyz')
            ccsegm = vigra.analysis.labelVolumeWithBackground(
                gcsegm.astype(np.uint8))

            # Extended bboxes of different objects might overlap.
            # To avoid conflicting segmentations, we find all connected
            # components in the results and only take the one, which
            # overlaps with the object "core" or "seed", defined by the
            # pre-thresholding
            seed = ccbox == i
            filtered = seed*ccsegm
            passed = vigra.analysis.unique(filtered.astype(np.uint32))
            assert len(passed.shape) == 1
            if passed.size > 2:
                logger.warn("ambiguous label assignment for region {}".format(
                    (xmin, xmax, ymin, ymax, zmin, zmax)))
                resbox[ccbox == i] = 1
            elif passed.size <= 1:
                logger.warn(
                    "box {} segmented out with beta {}".format(i, beta))
            else:
                # assign to the overlap region
                label = passed[1]  # 0 is background
                resbox[ccsegm == label] = 1

        pool = RequestPool()
        #FIXME make sure that the parallel computations fit into memory
        for i in range(1, nobj):
            req = Request(functools.partial(processSingleObject, i))
            pool.add(req)

        logger.info("Processing {} objects ...".format(nobj-1))

        pool.wait()
        pool.clean()

        logger.info("object loop done")

        # prepare result
        resView = vigra.taggedView(result, axistags=self.Output.meta.axistags)
        resView = resView.withAxes(*'xyz')

        # some labels could have been removed => relabel
#.........这里部分代码省略.........