Python Utility.report_status方法代码示例

# 需要导入模块: from utility import Utility [as 别名]
# 或者: from utility.Utility import report_status [as 别名]
 def reportTrainingStats(self, elapsedTime, batchIndex, valLoss, trainCost, mode=0):
     DB.storeTrainingStats( self.id, valLoss, trainCost, mode=mode)
     msg = '(%0.1f)     %i     %f%%'%\
     (
        elapsedTime,
        batchIndex,
        valLoss
     )
     status = '[%f]'%(trainCost)
     Utility.report_status( msg, status )

# 需要导入模块: from utility import Utility [as 别名]
# 或者: from utility.Utility import report_status [as 别名]
 def report(self, name, dims):
         msg = '%s'%(name)
         status = '['
         p_dim = None
         for dim in dims:
                 if p_dim == None:
                         status = '%s%s'%(status, dim)
                 else:
                         status = '%s x %s'%(status, dim)
                 p_dim  = dim
         status = '%s]'%status
         Utility.report_status( msg, status)

# 需要导入模块: from utility import Utility [as 别名]
# 或者: from utility.Utility import report_status [as 别名]
    def report_stats(self, id, elapsedTime, batchIndex, valLoss, trainCost, mode=0):

        if mode == 0 and not self.offline:
            DB.storeTrainingStats( id, valLoss, trainCost, mode=mode)
            self.add_validation_loss( valLoss )

        msg = '(%0.1f)     %i     %f%%'%\
        (
           elapsedTime,
           batchIndex,
           valLoss
        )
        status = '[%f]'%(trainCost)
        Utility.report_status( msg, status )

# 需要导入模块: from utility import Utility [as 别名]
# 或者: from utility.Utility import report_status [as 别名]
    def run(self):
        while not self.done: 
            # always query for the most current projects
            projects  = DB.getProjects()
            running   = None
            runningId = None

            # the first active project is the running project,
            # all others are deactivated 
            for project in projects:
                projectId     = project.id
                projectStatus = project.trainingStatus
                print project.startTime, project.id, projectStatus
                projectStatusStr = DB.projectStatusToStr( projectStatus )
                if projectStatus >= 1:
                    if running == None:
                        running = project
                        runningId = projectId
                    else:
                        print 'shutting down....', projectId
                        print projectStatusStr
                        print projectStatus
                        print runningId
                        DB.stopProject( projectId )
                        msg1 = 'stopping (%s)' %(projectId)
                        msg2 = '(%s) -> (Deactive)'%(projectStatusStr)
                        Utility.report_status( msg1, msg2 )

            # start the new project if changed.
            if self.projectId != runningId and running != None:
                projectStatus = running.trainingStatus
                projectStatusStr = DB.projectStatusToStr( projectStatus )
                self.create_model( running )

                # reset the training stats
                #if self.name == 'training':
                #	DB.removeTrainingStats( projectId )

                msg1 = 'starting (%s)' %(runningId)
                msg2 = '(Deactive) -> (Active)'
                Utility.report_status( msg1, msg2 )

            if self.model is not None:
                self.projectId = runningId
                self.work( running )
            time.sleep(self.waittime)
            print 'slept....', self.waittime

# 需要导入模块: from utility import Utility [as 别名]
# 或者: from utility.Utility import report_status [as 别名]
    def work_offline(self, project):
        
        imagePaths = sorted( glob.glob( '%s/*.tif'%(Paths.TrainGrayscale)  ) )

        for path in imagePaths:
            if self.done:
                break

            name = Utility.get_filename_noext( path )
          
            print 'path:', path 
            Utility.report_status('segmenting', '%s'%(name))

            #segPath = '%s/%s.offline.seg'%(Paths.TrainGrayscale, name)
            segPath = '%s/%s.%s.offline.seg'%(Paths.Segmentation, name, project.id)

            self.classify_n_save( path, segPath, project )

# 需要导入模块: from utility import Utility [as 别名]
# 或者: from utility.Utility import report_status [as 别名]
    def save_stats(self):

        n_data = len(self.p)        
        n_good = len( np.where( self.p == 0 )[0] )
        self.accuracy = float(n_good)/n_data

        print '------data.save_stats-----'
        print 'accuracy:', self.accuracy


        Utility.report_status('.', '.')
        for entry in self.entries:
            i = np.arange( entry.offset, entry.offset+entry.length )
            #y = self.y[ i ]
            p = self.p[ i ]
            n_data = len(p)
            n_good = len( np.where( p == 0 )[0] )
            score = 0.0 if n_good == 0 else float(n_good)/n_data

            #print np.bincount( self.p ), np.bincount( p ), n_good
            #print len(p), '/', len(self.p)

            DB.storeTrainingScore( self.project.id, entry.name, score )
            Utility.report_status('%s'%(entry.name), '%.2f'%(score))
            #print 'image (%s)(%.2f)'%(entry.name, score)
        Utility.report_status('.', '.')

# 需要导入模块: from utility import Utility [as 别名]
# 或者: from utility.Utility import report_status [as 别名]
    def work(self, project):

        if not self.online:
            self.work_offline(project)
            self.done = True
            return

        start_time = time.clock()

        if project is None:
            return

        print 'prediction.... running', len(self.high)

        if len(self.high) == 0:
            self.high = DB.getPredictionImages( project.id, 1)


        #FG - march 4th 2016
        #if len(self.low) == 0:
        #    self.low = DB.getPredictionImages( project.id, 0 )

        '''
        for img in self.high:
            print 'hid:', img.id, img.modelModifiedTime, img.segmentationTime

        print '----'
        for img in self.low:
            print 'lid:', img.id, img.modelModifiedTime, img.segmentationTime

        exit(1)
        '''

        task = None
        if (self.priority == 0 or len(self.low) == 0) and len(self.high) > 0:
            self.priority = 1
            task = self.high[0]
            del self.high[0]
        elif len(self.low) > 0:
            self.priority = 0
            task = self.low[0]
            del self.low[0]

        if task == None:
            return

        has_new_model = (self.modTime != project.modelTime)
        revision = DB.getRevision( project.id )
        print 'revision:', revision
        #has_new_model = (revision != self.revision or has_new_model)

        # reload the model if it changed
        if has_new_model:
            #self.revision = revision
            print 'initializing...'
            self.model.initialize()
            self.modTime = project.modelTime

        # read image to segment
        basepath = Paths.TrainGrayscale if task.purpose == 0 else Paths.ValidGrayscale
        path = '%s/%s.tif'%(basepath, task.id)
        #success, image = Utility.get_image_padded(path, project.patchSize ) #model.get_patch_size())

        print 'segment - path:', path
        print 'priority - ', task.segmentationPriority
        # perform segmentation

        Utility.report_status('segmenting %s'%(task.id),'')
        #probs = self.model.predict( path )
        #probs = self.model.classify( image )

        # serialize to file
        segPath = '%s/%s.%s.seg'%(Paths.Segmentation, task.id, project.id)
        #self.save_probs( probs, project.id, task.id )
        self.classify_n_save( path, segPath, project )         

        end_time = time.clock()
        duration = (end_time - start_time)
        DB.finishPrediction( self.projectId, task.id, duration, self.modTime )

# 需要导入模块: from utility import Utility [as 别名]
# 或者: from utility.Utility import report_status [as 别名]
        self.save_probs( prob, segPath)

    #-------------------------------------------------------------------
    # save probability map
    #-------------------------------------------------------------------
    def save_probs(self, data, path):
        output = StringIO.StringIO()
        output.write(data.tolist())
        content = output.getvalue()
        encoded = base64.b64encode(content)
        compressed = zlib.compress(encoded)
        with open(path, 'w') as outfile:
            outfile.write(compressed)

			
manager = None
def signal_handler(signal, frame):
        if manager is not None:
                manager.shutdown()

#---------------------------------------------------------------------------
# Entry point to the main function of the program.
#---------------------------------------------------------------------------
if __name__ == '__main__':
    print sys.argv
    Utility.report_status('running prediction module', '')
    signal.signal(signal.SIGINT, signal_handler)

    manager = Prediction()
    Manager.start( sys.argv, manager )

# 需要导入模块: from utility import Utility [as 别名]
# 或者: from utility.Utility import report_status [as 别名]
    def load_validation(self):

        # retrieve the list of training images 
        # (annotated images)
        valid_new = len(self.entries_valid) > 0
        print 'valid_new: ', valid_new
        images = DB.getImages( self.project.id, purpose=1, new=valid_new, annotated=True )

        # bailout if there's no images to train.
        if len(images) == 0:
            return

        # determine the maximum number of samples to draw
        # from each image
        n_samples_per_image = Data.MaxSamples/len(images)

        print '#n_samples_per_image:', n_samples_per_image
        print '#images:', len(images)

        entries = []

        # Load training samples for each image.
        for image in images:

            Utility.report_status( 'loading validation image', image.id)
            print 'ttime:', image.trainingTime
            print 'atime:', image.annotationTime
            print 'tstat:', image.trainingStatus

            offset = len( entries )

            # generate samples for the image
            #data   = self.gen_samples( project, image.id, n_samples_per_image )
            data   = self.gen_samples( Paths.ValidGrayscale, self.project, image.id, n_samples_per_image )
            x_data = data[0]
            y_data = data[1]
            n_data = len( y_data )

            # skip if no annotations found
            if n_data == 0:
                continue

            # add sample to the training set
            if offset == 0:
                x = x_data
                y = y_data
                p = np.ones( n_data, dtype=np.int )
            else:
                x = np.vstack( (x, x_data) )
                y = np.hstack( (y, y_data) )
                p = np.hstack( (p, np.ones( n_data, dtype=np.int )) )

            # keep track of each image's data in an entry for 
            # easier replacement.
            entries.append( Entry( image.id, offset, n_data ) )

            #Utility.report_memused()
            Utility.report_status('x', '(%d bytes)'%(x.nbytes))
            Utility.report_status('y', '(%d bytes)'%(y.nbytes))
            Utility.report_status('.','.')



        # bailout if no entries found
        if len(entries) == 0:
            Utility.report_status('Fetching new data', 'None Found')
            return

        Utility.report_status( 'Loading new data', 'please wait')

        # bailout if no current entries
        if len(self.entries_valid) > 0:
            #append old entries after the new entries
            offset = len(y)

            print entries[-1].name, entries[-1].offset, entries[-1].length
            mask = np.ones( len(self.y_valid), dtype=bool)
            names = [ e.name for e in entries ]

            for entry in self.entries_valid:
                if entry.name in names:
                    mask[ entry.offset : entry.offset+entry.length ] = False
                else:
                    entry.offset = offset
                    offset += entry.length
                    entries.append( entry )
                    print entry.name, entry.offset, entry.length

            x_keep = self.x_valid[ mask ]
            y_keep = self.y_valid[ mask ]
            p_keep = self.p_valid[ mask ]
            x = np.vstack( (x, x_keep) )
            y = np.hstack( (y, y_keep) )
            p = np.hstack( (p, p_keep) )


        if len( np.unique( y ) ) <= 1:
            print 'not enough labels specified...'
            return

#.........这里部分代码省略.........

# 需要导入模块: from utility import Utility [as 别名]
# 或者: from utility.Utility import report_status [as 别名]
    def load_training(self):

        

        # retrieve the list of training images 
        # (annotated images)
        first_time = (len(self.entries) == 0)
        images     = DB.getTrainingImages( self.project.id, new=(not first_time) )
        imgs = DB.getImages( self.project.id )

        # bailout if there's no images to train.
        if len(images) == 0:
            return

        # determine the maximum number of samples to draw
        # from each image
        n_samples_per_image = Data.MaxSamples/len(images)

        print '#n_samples_per_image:', n_samples_per_image
        print '#images:', len(images)

        entries = []

        # Load training samples for each image.
        for image in images:

            Utility.report_status( 'loading', image.id)
            print 'ttime:', image.trainingTime
            print 'atime:', image.annotationTime
            print 'tstat:', image.trainingStatus

            offset = len( entries )

            # generate samples for the image
            #data   = self.gen_samples( project, image.id, n_samples_per_image )
            data   = self.gen_samples( Paths.TrainGrayscale, self.project, image.id, n_samples_per_image )
            x_data = data[0]
            y_data = data[1]
            n_data = len( y_data )

            print 'wmean:', data[2], 'wstd:', data[3], 'mean:', self.project.mean, 'std:', self.project.std

            # skip if no annotations found
            if n_data == 0:
                continue

            # add sample to the training set
            if offset == 0:
                x = x_data
                y = y_data
                p = np.ones( n_data, dtype=np.int )
            else:
                x = np.vstack( (x, x_data) )
                y = np.hstack( (y, y_data) )
                p = np.hstack( (p, np.ones( n_data, dtype=np.int )) ) 

            # keep track of each image's data in an entry for 
            # easier replacement.
            entries.append( Entry( image.id, offset, n_data ) )

            #Utility.report_memused()
            Utility.report_status('x', '(%d bytes)'%(x.nbytes))
            Utility.report_status('y', '(%d bytes)'%(y.nbytes))
            Utility.report_status('.','.')


        # bailout if no entries found
        if len(entries) == 0:
            Utility.report_status('Fetching new data', 'None Found')
            return

        Utility.report_status( 'Loading new data', 'please wait')

        # bailout if no current entries
        if len(self.entries) > 0:
            #append old entries after the new entries
            offset = len(y)

            print entries[-1].name, entries[-1].offset, entries[-1].length
            mask = np.ones( len(self.y), dtype=bool)
            names = [ e.name for e in entries ]

            for entry in self.entries:
                if entry.name in names:
                    mask[ entry.offset : entry.offset+entry.length ] = False
                else:
                    entry.offset = offset
                    offset += entry.length
                    entries.append( entry )
                    print entry.name, entry.offset, entry.length

            x_keep = self.x[ mask ]
            y_keep = self.y[ mask ]
            p_keep = self.p[ mask ]
            x = np.vstack( (x, x_keep) )
            y = np.hstack( (y, y_keep) )
            p = np.hstack( (p, p_keep) )

        
        if len( np.unique( y ) ) <= 1:
#.........这里部分代码省略.........

# 需要导入模块: from utility import Utility [as 别名]
# 或者: from utility.Utility import report_status [as 别名]
    def aload(self, project):

        self.projecft = project

        # LOAD TRAINING DATA 
        train_new = len(self.entries) > 0
        images = DB.getImages( project.id, purpose=0, new=train_new)
        out = self.load_data(
                Paths.TrainGrayscale, 
                images, 
                project, 
                self.x, 
                self.y,
                self.p, 
                self.entries)

        self.x = out[0]
        self.y = out[1]
        self.p = out[2]
        self.entries = out[3]

        n_samples = len(self.y)
        self.i = np.arange( n_samples )
        self.n_superbatch = int(n_samples/(Data.TrainSuperBatchSize + Data.ValidSuperBatchSize))
        self.i_randomize  = 0 
        self.data_changed = True
        self.i_train = []
        self.avg_losses = []
        self.last_avg_loss = 0 

        if n_samples >  0:  
            Utility.report_status('---------training---------', '') 
            Utility.report_status('#samples','(%d)'%len(self.y))
            Utility.report_status('x shape','(%d,%d)'%(self.x.shape[0], self.x.shape[1]))
            Utility.report_status('y shape','(%d)'%(self.x.shape[0]))
            Utility.report_status('x memory', '(%d bytes)'%(self.x.nbytes))
            Utility.report_status('y memory', '(%d bytes)'%(self.y.nbytes))

        print 'min:', np.min( self.x )
        print 'max:', np.max( self.x )
        print 'uy:', np.unique( self.y )
        print 'x:', self.x[:5]
        print 'y:', self.y[:5]


        # LOAD VALIDATION IMAGES
        valid_new = len(self.entries_valid) > 0
        images = DB.getImages( project.id, purpose=1, new=valid_new )
        out = self.load_data(   
                Paths.ValidGrayscale, 
                images, 
                project, 
                self.x_valid, 
                self.y_valid, 
                self.p_valid,
                self.entries_valid)

        self.x_valid = out[0]
        self.y_valid = out[1]
        self.p_valid = out[2]
        self.entries_valid = out[3]

        n_samples = len(self.y_valid)
        self.i_valid = np.arange( n_samples )

        if n_samples >  0:
            Utility.report_status('---------validation---------', '')
            Utility.report_status('#samples','(%d)'%len(self.y_valid))
            Utility.report_status('x shape','(%d,%d)'%(self.x_valid.shape[0], self.x_valid.shape[1]))
            Utility.report_status('y shape','(%d)'%(self.x_valid.shape[0]))
            Utility.report_status('x memory', '(%d bytes)'%(self.x_valid.nbytes))
            Utility.report_status('y memory', '(%d bytes)'%(self.y_valid.nbytes))

        print 'min:', np.min( self.x_valid )
        print 'max:', np.max( self.x_valid )
        print 'uy:', np.unique( self.y_valid )
        print 'x:', self.x_valid[:5]
        print 'y:', self.y_valid[:5]

        Utility.report_memused()
        DB.finishLoadingTrainingset( project.id )

# 需要导入模块: from utility import Utility [as 别名]
# 或者: from utility.Utility import report_status [as 别名]
        def predict(self):
		Utility.report_status('starting prediction', '')

		msg = 'loading prediction data'
		if not self.datasets.load_test():
			Utility.report_status( msg, 'fail')
			return False

		Utility.report_status( msg, 'done' )

                self.report( 'test set x', \
                self.datasets.test_set_x.shape.eval())
                self.report( 'test set y', \
                self.datasets.test_set_y.shape.eval())

                path = '%s/best_model.pkl'%(self.settings.params)
                msg = 'loading best model'
                if not os.path.exists(path):
                        Utility.report_status( msg, 'fail' )
                        return False

                n_hidden = self.settings.n_hidden
                n_classes = self.datasets.n_classes
                n_features = self.datasets.n_features

		w_val = np.random.normal(0, 0.1, (n_features, n_hidden))
		b_val = np.zeros( n_hidden )
                w = theano.shared(value=w_val, name='W', borrow=True)
                b = theano.shared(value=b_val, name='b', borrow=True)
                save_file = open(path)
		w_val, b_val = cPickle.load(save_file)
                w.set_value(w_val)
		#cPickle.load(save_file), borrow=True)
                b.set_value(b_val)
		#cPickle.load(save_file), borrow=True)

		print 'w:', w.shape.eval()
		print 'b:', b.shape.eval()

                Utility.report_status( msg, 'done' )


		test_set_x = self.datasets.test_set_x
		test_set_y = self.datasets.test_set_y

		rng = np.random.RandomState(42)

                # Step 1. Declare Theano variables
                x = T.fmatrix()
                y = T.ivector()
                index = T.iscalar()

		path = '%s/%s'%(self.settings.models, 'best_model_mlp.pkl')
		print 'loading %'%(path)
		classifier = MLP_Ex( path )
		if True:
			return True

                classifier = MLP(
                        rng=rng,
                        input=x,
                        n_in=n_features,
                        n_hidden=n_hidden,
                        n_out=n_classes
                )   
                classifier.params = []
		classifier.params.extend([ w, b ])

                predict_model = theano.function(
                    inputs=[index],
                    outputs=classifier.logRegressionLayer.y_pred,
                    givens={
                            x: test_set_x,
                            y: test_set_y
                        },
		    on_unused_input='ignore' 
                    )   

		predicted_values = predict_model( 1 )
		print 'size predicted:', len( predicted_values )

		msg = 'image segmentation'
		Utility.report_status( msg, 'done' )
 		return True	

# 需要导入模块: from utility import Utility [as 别名]
# 或者: from utility.Utility import report_status [as 别名]
        print 'path:', path
        print 'images:'
        print [img.id for img in images]
              
        if len(images) == 0:
            return x_out, y_out, p_out, entries_out

        # determine the maximum number of samples to draw
        # from each image
        n_samples_per_image = Data.MaxSamples/len(images)


        entries = []

        for image in images:
            Utility.report_status( 'loading', image.id)
            print 'ttime:', image.trainingTime
            print 'atime:', image.annotationTime
            print 'tstat:', image.trainingStatus

            offset = len( entries )

            # generate samples for the image
            data   = self.gen_samples( path, project, image.id, n_samples_per_image )
            x_data = data[0]
            y_data = data[1]
            n_data = len( y_data )

            # skip if no annotations found
            if n_data == 0:
                continue

# 需要导入模块: from utility import Utility [as 别名]
# 或者: from utility.Utility import report_status [as 别名]

#.........这里部分代码省略.........

        updates = gradient_updates_momentum(cost, self.params, lr, m)

        train_model = theano.function(inputs=[index], outputs=cost,
                updates=updates,
                givens={
                    x: train_set_x[index * batchSize:(index + 1) * batchSize],
                    y: train_set_y[index * batchSize:(index + 1) * batchSize],
                    lr: lr_shared,
                    m: m_shared})


        # TRAIN THE MODEL
        print '... training'
        print 'self.best_validation_loss:', self.best_validation_loss
        best_iter = 0
        validation_frequency = 1

        start_time = time.clock()

        elapsed_time = 0
        iter = 0

        minibatch_avg_costs = []
        minibatch_index = 0


        #while (elapsed_time < self.trainTime)\
        #    and (minibatch_index<n_train_batches)\
        #    and (not self.done):
        while (minibatch_index<n_train_batches) and (not self.done):
            if (elapsed_time >= self.trainTime):
                break

            train_cost = train_model(minibatch_index)

            # test the trained samples against the target
            # values to measure the training performance
            i = minibatch_index

            '''
            probs = predict_samples(minibatch_index)
            #print 'probs:', probs.shape
            i_batch = data.i_train[ i * batchSize:(i+1)*batchSize ]
            data.p[ i_batch ] = probs
            '''

            '''
            good = np.where( probs == 0)[0]
            bad  = np.where( probs == 1)[0]
            print 'bad:', len(bad)
            print 'good:', len(good)
            #print probs
            '''
            #print '----->traincost:', type(train_cost), train_cost

            minibatch_avg_costs.append(train_cost)

            iter += 1
            #iter = (epoch - 1) * n_train_batches + minibatch_index
            if (iter + 1) % validation_frequency == 0 and valid_samples > 0:

                validation_losses = np.array([validate_model(i) for i in xrange(n_valid_batches)])
                this_validation_loss = numpy.sum(validation_losses) * 100.0 / valid_samples
                elapsed_time = time.clock() - start_time

                '''
                self.reportTrainingStats(elapsed_time,
                        minibatch_index,
                        this_validation_loss,
                        minibatch_avg_costs[-1].item(0))
                '''
                print this_validation_loss, '/', self.best_validation_loss
                data.add_validation_loss( this_validation_loss )

                # if we got the best validation score until now
                if this_validation_loss < self.best_validation_loss:
                    self.best_validation_loss = this_validation_loss
                    best_iter = iter

                    self.save()
                    print "New best score!"

            # advance to next mini batch
            minibatch_index += 1

            # update elapsed time
            elapsed_time = time.clock() - start_time

        if valid_samples == 0:
            self.save()

        probs = predict_samples()
        data.p[ data.i_train ] = probs

        elapsed_time = time.clock() - start_time
        msg = 'The code an for'
        status = '%f seconds' % (elapsed_time)
        Utility.report_status( msg, status )
        print 'done...'

# 需要导入模块: from utility import Utility [as 别名]
# 或者: from utility.Utility import report_status [as 别名]
	def report(self):
	    Utility.report_status('learning rate', "%s"%self.learning_rate)
	    Utility.report_status('hidden units', "%s"%self.n_hidden)
	    Utility.report_status('sample_size', "%s"%self.sample_size)