Python SupervisedDataSet.splitWithProportion方法代码示例

# 需要导入模块: from pybrain.datasets import SupervisedDataSet [as 别名]
# 或者: from pybrain.datasets.SupervisedDataSet import splitWithProportion [as 别名]
    def handle(self, *args, **options):
        better_thans = BetterThan.objects.all() #.filter(pk__lte=50)

        ds = SupervisedDataSet(204960, 1)
        for better_than in better_thans:
            bt = imread(better_than.better_than.image.file)
            wt = imread(better_than.worse_than.image.file)
            better_than.better_than.image.file.close()
            better_than.worse_than.image.file.close()

            # bt = filters.sobel(bt)
            # wt = filters.sobel(wt)

            bt_input_array = np.reshape(bt, (bt.shape[0] * bt.shape[1]))
            wt_input_array = np.reshape(wt, (wt.shape[0] * wt.shape[1]))
            input_1 = np.append(bt_input_array, wt_input_array)
            input_2 = np.append(wt_input_array, bt_input_array)
            ds.addSample(np.append(bt_input_array, wt_input_array), [-1])
            ds.addSample(np.append(wt_input_array, bt_input_array), [1])
        
        net = buildNetwork(204960, 2, 1)

        train_ds, test_ds = ds.splitWithProportion(options['train_test_split'])
        _, test_ds = ds.splitWithProportion(options['test_split'])

        trainer = BackpropTrainer(net, ds)

        print 'Looking for -1: {0}'.format(net.activate(np.append(bt_input_array, wt_input_array)))
        print 'Looking for 1: {0}'.format(net.activate(np.append(wt_input_array, bt_input_array)))

        trainer.train()

        print 'Looking for -1: {0}'.format(net.activate(np.append(bt_input_array, wt_input_array)))
        print 'Looking for 1: {0}'.format(net.activate(np.append(wt_input_array, bt_input_array)))

# 需要导入模块: from pybrain.datasets import SupervisedDataSet [as 别名]
# 或者: from pybrain.datasets.SupervisedDataSet import splitWithProportion [as 别名]
def makeNet(learning_rate):
	ds = SupervisedDataSet(20, 20)
	with open('data/misspellingssmall.csv', 'rbU') as f:
		reader = csv.reader(f)
		for row in reader:
			ds.addSample(convert(row[0]),convert(row[1]))

	testds, trainds = ds.splitWithProportion(0.2)

	net = buildNetwork(20, 20, 20)
	trainer = BackpropTrainer(net, dataset=trainds, learningrate=learning_rate)
	
	myscore = float("inf")
	i = 0
	while myscore > 5:
		i += 1

		trainer.train()
		#trainer.trainEpochs(5)
		#trainer.trainUntilConvergence(verbose=True)

		myscore = score(net, testds)
		print "Epoch #" + str(i) + ": " + str(myscore) + " (" + unconvert(net.activate(convert("ecceptable"))) + ")"

	global lastNet
	lastNet = net

	print "Network done with score " + str(myscore)
	
	return score

# 需要导入模块: from pybrain.datasets import SupervisedDataSet [as 别名]
# 或者: from pybrain.datasets.SupervisedDataSet import splitWithProportion [as 别名]
def buildDataset(inpts, targets):
    i = 0
    ds = SupervisedDataSet(12, 1)
    while i != len(inpts):
        ds.addSample(inpts[i], targets[i])
        i = i + 1
    return ds.splitWithProportion(0.75)

# 需要导入模块: from pybrain.datasets import SupervisedDataSet [as 别名]
# 或者: from pybrain.datasets.SupervisedDataSet import splitWithProportion [as 别名]
 def _buildDataset(self, inpts, targets):
     i = 0
     ds = SupervisedDataSet(len(inpts[0]), len(targets[0]))
     while i != len(inpts):
         ds.addSample(inpts[i], targets[i])
         i = i + 1
     return ds.splitWithProportion(1)

# 需要导入模块: from pybrain.datasets import SupervisedDataSet [as 别名]
# 或者: from pybrain.datasets.SupervisedDataSet import splitWithProportion [as 别名]
class NNet(object):
    def __init__(self):
        self.net = buildNetwork(2, 4, 2, bias=True)
        self.net.randomize()
        print self.net
        self.ds = SupervisedDataSet(2,2)
        self.trainer = BackpropTrainer(self.net, self.ds, learningrate = 0.1, momentum=0.99)
    def addTrainDS(self, data1, data2, max):
        for x in [1,2]:
            norm1 = self.normalize(data1,max)
            norm2 = self.normalize(data2,max)
        for x in range(len(norm1)):
            self.ds.addSample(norm1[x], norm2[x])
    def train(self):
        print "Training"
        # print self.trainer.train()
        trndata, tstdata = self.ds.splitWithProportion(.25)
        self.trainer.trainUntilConvergence(verbose=True,
                                           trainingData=trndata,
                                           validationData=tstdata,
                                           validationProportion=.3,
                                           maxEpochs=500)
        # self.trainer.trainOnDataset(trndata,500)
        self.trainer.testOnData(tstdata, verbose= True)

    def activate(self, data):
        for x in data:
            self.net.activate(x)

    def normalize(self, data, max):
        normData = np.zeros((len(data), 2))
        for x in [0,1]:
            for y in range(len(data)):
                val = data[y][x]
                normData[y][x] = (val)/(max[x])
        # print normData
        return normData

    def denormalize(self, data, max):
        deNorm = np.zeros((len(data), 2))
        for x in [0,1]:
            for y in range(len(data)):
                val = data[y][x]
                deNorm[y][x] = val*max[x]
        return deNorm

    def getOutput(self, mat, max):
        norm = self.normalize(mat, max)
        out = []
        for val in norm:
            out.append(self.net.activate(val))
        return self.denormalize(out, max)

# 需要导入模块: from pybrain.datasets import SupervisedDataSet [as 别名]
# 或者: from pybrain.datasets.SupervisedDataSet import splitWithProportion [as 别名]
def learn(input, output):
    """
    Learn nn from data.
    """
    nn = RecurrentNeuralNetwork(13, 4)
    dataset = SupervisedDataSet(13, 4)
    for ins, out in zip(input, output):
        dataset.addSample(ins, out)

    learning, validating = dataset.splitWithProportion(0.8)
    nn.set_learning_data(learning)
    nn.train(75)

    result = nn.calculate(validating)

    return result, validating['target']

# 需要导入模块: from pybrain.datasets import SupervisedDataSet [as 别名]
# 或者: from pybrain.datasets.SupervisedDataSet import splitWithProportion [as 别名]
def neural_network_converg(data, target, network):
    DS = SupervisedDataSet(len(data[0]), 1)
    nn = buildNetwork(len(data[0]), 7, 1, bias = True, hiddenclass = SigmoidLayer, outclass = LinearLayer) 
    for d, t in zip(data, target):
         DS.addSample(d,t)
    Train, Test = DS.splitWithProportion(0.9)
    #data_train = Train['input']
    data_test = Test['input']
    #target_train = Train['target']
    target_test = Test['target']
    bpTrain = BackpropTrainer(nn,Train, verbose = True)
    #bpTrain.train()
    bpTrain.trainUntilConvergence(maxEpochs = 10)
    p = []
    for d_test in data_test:
        p.append(nn.activate(d_test))
        
    rmse_nn = sqrt(np.mean((p - target_test)**2)) 
    print(rmse_nn) 

# 需要导入模块: from pybrain.datasets import SupervisedDataSet [as 别名]
# 或者: from pybrain.datasets.SupervisedDataSet import splitWithProportion [as 别名]
    def build_pybrain_dataset(self):
        field_count = len(dataset.fields)

        diagnoses_count = len(self.diagnoses)

        supervised_dataset = SupervisedDataSet(field_count, diagnoses_count)
        # supervised_dataset = ClassificationDataSet(field_count,
        #                                            diagnoses_count,
        #                                            nb_classes=diagnoses_count,
        #                                            class_labels=self.diagnoses)

        for sample in self.data:
            input = self.make_input(sample)
            diagnosis = sample['Диагноз']
            target = self.make_target(diagnosis)
            supervised_dataset.addSample(input, target)

        self.supervised_dataset = supervised_dataset
        # self.training_dataset = supervised_dataset
        # self.testing_dataset = supervised_dataset
        self.training_dataset, self.testing_dataset = supervised_dataset.splitWithProportion(0.7)

# 需要导入模块: from pybrain.datasets import SupervisedDataSet [as 别名]
# 或者: from pybrain.datasets.SupervisedDataSet import splitWithProportion [as 别名]
def vali():
    from pybrain.tools.validation import ModuleValidator
    from pybrain.tools.validation import CrossValidator
    with open('new_data1.txt') as data_file:
        data = json.load(data_file)
    m = [d[0] for d in data]
    case = [min([a for a, s, d in m]), float(max([a for a, s, d in m])-min([a for a, s, d in m]))]
    week = [min([s for a, s, d in m]), float(max([s for a, s, d in m])-min([s for a, s, d in m]))]
    grid = [min([d for a, s, d in m]), float(max([d for a, s, d in m])-min([d for a, s, d in m]))]
    ds = SupervisedDataSet(3, 1)
    import random
    random.shuffle(data)
    print len(data)
    for i in xrange(0, len(data)):
        # print "Adding {}th data sample".format(i),
        x1 = float(data[i][0][0] - case[0])/case[1]
        x2 = float(data[i][0][1] - week[0])/week[1]
        x3 = float(data[i][0][2] - grid[0])/grid[1]
        input = (x1, x2, x3)
        output = data[i][1]
        ds.addSample(input, output)
        # print ":: Done"

    print "Train"
    net = buildNetwork(3, 3, 1, bias=True)
    tstdata, trndata = ds.splitWithProportion( 0.33 )
    trainer = BackpropTrainer(net, trndata)
    mse = []
    modval = ModuleValidator()
    for i in range(100):
        trainer.trainEpochs(1)
        trainer.trainOnDataset(dataset=trndata)
        cv = CrossValidator(trainer, trndata, n_folds=10, valfunc=modval.MSE)
        mse_val = cv.validate()
        print "MSE %f @ %i" % (mse_val, i)
        mse.append(mse_val)

    with open('cross_validation.json', 'w') as outfile:
            json.dump(mse, outfile, indent=4)

# 需要导入模块: from pybrain.datasets import SupervisedDataSet [as 别名]
# 或者: from pybrain.datasets.SupervisedDataSet import splitWithProportion [as 别名]
    def classicNeuralNetwork(self,features,labels,autoencoder=False):
        dataSet = SupervisedDataSet(features.shape[1], 1)
        dataSet.setField('input', features)
        if autoencoder: labels = features      
        dataSet.setField('target', labels)
        tstdata, trndata = dataSet.splitWithProportion( 0.25 )
        print features.shape
        simpleNeuralNetwork = _buildNetwork(\
                                    (LinearLayer(features.shape[1],'in'),),\
                                    (SigmoidLayer(20,'hidden0'),),\
                                    (LinearLayer(labels.shape[1],'out'),),\
                                    bias=True)
        trainer = BackpropTrainer(simpleNeuralNetwork, dataset=trndata, verbose=True)#, momentum=0.1)
        trainer.trainUntilConvergence(maxEpochs=15)
        
        trnresult = percentError( trainer.testOnData( dataset=trndata ), trndata['target'] )
        tstresult = percentError( trainer.testOnData( dataset=tstdata ), tstdata['target'] )

        print "epoch: %4d" % trainer.totalepochs, \
          "  train error: %5.2f%%" % trnresult, \
          "  test error: %5.2f%%" % tstresult

        self.neuralNetwork = simpleNeuralNetwork

# 需要导入模块: from pybrain.datasets import SupervisedDataSet [as 别名]
# 或者: from pybrain.datasets.SupervisedDataSet import splitWithProportion [as 别名]
def run_data1():
    with open('new_data1.txt') as data_file:
        data = json.load(data_file)
    output = set([i[2] for i in [d[0] for d in data if d[1] == 1]])
    print output
    m = [d[0] for d in data]
    print (max([d for a, s, d in m]), min([d for a, s, d in m]), float(max([d for a, s, d in m])-min([d for a,s,d in m])))
    case = [min([a for a, s, d in m]), float(max([a for a, s, d in m])-min([a for a,s,d in m]))]
    week = [min([s for a, s, d in m]), float(max([s for a, s, d in m])-min([s for a,s,d in m]))]
    grid = [min([d for a, s, d in m]), float(max([d for a, s, d in m])-min([d for a,s,d in m]))]
    ds = SupervisedDataSet(3, 1)
    import random
    random.shuffle(data)
    print len(data)
    for i in xrange(0, len(data)):
        # print "Adding {}th data sample".format(i),
        x1 = float(data[i][0][0] - case[0])/case[1]
        x2 = float(data[i][0][1] - week[0])/week[1]
        x3 = float(data[i][0][2] - grid[0])/grid[1]
        input = (x1, x2, x3)
        output = data[i][1]
        ds.addSample(input, output)
        # print ":: Done"

    print "Train"
    # net = buildNetwork(3, 3, 1, bias=True)\
    net = NetworkReader.readFrom('dengue_network.xml')
    tstdata, trndata = ds.splitWithProportion( 0.33 )
    trainer = BackpropTrainer(net, trndata)
    # terrors = trainer.trainUntilConvergence(verbose = True, validationProportion = 0.33, maxEpochs = 100, continueEpochs = 10 )


    # mse = [0]
    # acceptable_error = .00001
    # for i in xrange(0,1000):
    #     print i," ",
    #     mse_c = trainer.train()
    #     if (mse_c < acceptable_error):
    #         break
    #     mse.append(mse_c)
    #     print mse_c

    threshold = [0.25, 0.30]
    for t in threshold:
        print "Testing threshold :", t
        true_positive = 0.0
        true_negative = 0.0
        false_positive = 0.0
        false_negative = 0.0

        data_to_write = []
        data_to_write_input = []
        for input, expectedOutput in tstdata:
            o = net.activate(input)
            output = 1.0 if o[0] > t else 0.0
            data_to_write.append((int((input[0]*case[1]) + case[0]), int((input[1]*week[1]) + week[0]),int((input[2]*grid[1]) + grid[0]), output))
            if (output == expectedOutput):
                if output == 1.0:
                    true_positive += 1.0
                else:
                    true_negative += 1.0
            else:
                if output == 1.0:
                    false_positive += 1.0
                else:
                    false_negative += 1.0
        # NetworkWriter.writeToFile(net, 'dengue_network1.xml')
        precision = true_positive / (true_positive + false_positive)
        recall = true_positive / (true_positive + false_negative)
        f = (2 * precision * recall)/(precision + recall)
        accuracy = (true_positive + true_negative) / (true_positive + true_negative + false_positive + false_negative)

        def getKey(item):
            return item[1]
        data_to_write = sorted(data_to_write,  key=getKey)
        counts = {
            # "MSE" : mse,
            # "DATA": data_to_write,
            "Threshold": t,
            "Precision": precision,
            "Recall": recall,
            "F-Measure": f,
            "Accuracy": accuracy,
            "Values": {
                "True Positive": true_positive,
                "True Negative": true_negative,
                "False Positive": false_positive,
                "False Negative": false_negative
            }
        }
        print "Accuracy :", accuracy
        print "Precision :", precision
        print "Recall :", recall
        print "F-Measure :", f
        print counts
        # errors = {
        #     "terrors" : terrors
        # }
        # with open('data8.json', 'w') as outfile:
        #     json.dump(counts, outfile, indent=4)
#.........这里部分代码省略.........

# 需要导入模块: from pybrain.datasets import SupervisedDataSet [as 别名]
# 或者: from pybrain.datasets.SupervisedDataSet import splitWithProportion [as 别名]
def brescia_nn(train, test, max_epochs=None, verbose=False):
    trainval_ds = SupervisedDataSet(5, 1)
    test_ds = SupervisedDataSet(5, 1)
    
    for datum in train:
        trainval_ds.addSample(datum[:5], (datum[5],))

    for datum in test:
        test_ds.addSample(datum[:5], (datum[5],))
    
    train_ds, val_ds = trainval_ds.splitWithProportion(0.75)
    
    if verbose:
        print "Train, validation, test:", len(train_ds), len(val_ds), len(test_ds)
    
    ns = {}
    min_error = -1
    min_h = -1
    
    # use validation to form 4-layer network with two hidden layers,
    # with (2n + 1) nodes in the first hidden layer and somewhere from
    # 1 to (n - 1) in the second hidden layer
    for h2 in range(1, 5):
        if verbose:
            start = time.time()
            print "h2 nodes:", h2
    
        # create the network
        if verbose:
            print "building network"

        n = FeedForwardNetwork()
        inLayer = LinearLayer(5)
        hiddenLayer1 = SigmoidLayer(11)
        hiddenLayer2 = SigmoidLayer(h2)
        outLayer = LinearLayer(1)
    
        n.addInputModule(inLayer)
        n.addModule(hiddenLayer1)
        n.addModule(hiddenLayer2)
        n.addOutputModule(outLayer)
    
        in_to_hidden = FullConnection(inLayer, hiddenLayer1)
        hidden_to_hidden = FullConnection(hiddenLayer1, hiddenLayer2)
        hidden_to_out = FullConnection(hiddenLayer2, outLayer)
    
        n.addConnection(in_to_hidden)
        n.addConnection(hidden_to_hidden)
        n.addConnection(hidden_to_out)
    
        n.sortModules()
    
        # training
        if verbose:
            print "beginning training"
        trainer = BackpropTrainer(n, train_ds)
        trainer.trainUntilConvergence(maxEpochs=max_epochs)

        ns[h2] = n
    
        # validation
        if verbose:
            print "beginning validation"

        out = n.activateOnDataset(val_ds)
        actual = val_ds['target']
        error = np.sqrt(np.sum((out - actual)**2) / len(val_ds))
        if verbose:
            print "RMSE:", error
    
        if min_error == -1 or error < min_error:
            min_error = error
            min_h = h2
    
        if verbose:
            stop = time.time()
            print "Time:", stop - start
    
    # iterate through
    if verbose:
        print "best number of h2 nodes:", min_h
    out_test = ns[min_h].activateOnDataset(test_ds)

    return ns[h2], out_test

# 需要导入模块: from pybrain.datasets import SupervisedDataSet [as 别名]
# 或者: from pybrain.datasets.SupervisedDataSet import splitWithProportion [as 别名]
num_trials, num_features = X_successful.shape
alldata = SupervisedDataSet(num_features,1) 

stimalldata = SupervisedDataSet(num_features,1)

# add the features and class labels into the dataset
for xnum in xrange(num_trials): 
    alldata.addSample(X_successful[xnum,:],y_successful[xnum])

# add the features and dummy class labels into the stim dataset
for xnum in xrange(len(ind_successful_stress_stim)):
	stimalldata.addSample(X_successful_stim[xnum,:],y_successful_stim[xnum])

# split the data into testing and training data
tstdata_temp, trndata_temp = alldata.splitWithProportion(0.15)

# small bug with _convertToOneOfMany function.  This fixes that
tstdata = ClassificationDataSet(num_features,1,nb_classes=2)
for n in xrange(0, tstdata_temp.getLength()):
    tstdata.addSample(tstdata_temp.getSample(n)[0], tstdata_temp.getSample(n)[1])

trndata = ClassificationDataSet(num_features,1,nb_classes=2)
for n in xrange(0,trndata_temp.getLength()):
    trndata.addSample(trndata_temp.getSample(n)[0],trndata_temp.getSample(n)[1])

valdata = ClassificationDataSet(num_features,1,nb_classes=2)
for n in xrange(0,stimalldata.getLength()):
    valdata.addSample(stimalldata.getSample(n)[0],stimalldata.getSample(n)[1])

# organizes dataset for pybrain

# 需要导入模块: from pybrain.datasets import SupervisedDataSet [as 别名]
# 或者: from pybrain.datasets.SupervisedDataSet import splitWithProportion [as 别名]
result_r   = rev_map(result)

# buying       vhigh, high, med, low
# maint        vhigh, high, med, low
# doors        2, 3, 4, 5more
# persons      2, 4, more
# lug_boot     small, med, big
# safety       low, med, high

with open(fname, "r") as f:
    reader = csv.reader(f)
    for row in reader:
        sample = (price[row[0]], price[row[1]], doors[row[2]], persons[row[3]], lug_boot[row[4]], safety[row[5]])
        ds.addSample(sample, result[row[6]])

tst_ds, trn_ds = ds.splitWithProportion(0.2)

# print "train data"
# for inpt, target in trn_ds:
#     print inpt, target

# print "test data"
# for inpt, target in tst_ds:
#     print inpt, target

# More information about trainers: http://pybrain.org/docs/api/supervised/trainers.html

print "Training started"

trainer.trainOnDataset(trn_ds, 10)

# 需要导入模块: from pybrain.datasets import SupervisedDataSet [as 别名]
# 或者: from pybrain.datasets.SupervisedDataSet import splitWithProportion [as 别名]
def calc():
    filePath = 'asc_gyro_l.skl'
    f = open(filePath, 'r')
    headers = f.readline().split()
    indices = [2]
    numOfFeatures = len(indices)#len(ancestorMap)
    ds = SupervisedDataSet(numOfFeatures, 1)
    press0 = []
    press1 = []
    for line in f:
        splited = line.split()
        output = [float(splited[2]) - 32920.0]#, splited[3]]
        press0.append(float(output[0]))
        #press1.append(float(output[1]))
        input = np.array(splited)
        input = input[indices]#getAnccestorRelativePos(splited)#splited[7:]#
        ds.appendLinked(output[0], output)
    tstdata, trndata = ds.splitWithProportion( 0.25 )
    
    #for n in range(5):
    numOfHidden = 1#15*n + 1
    net = buildNetwork(numOfFeatures, numOfHidden, 1, bias=True)
    #net = FeedForwardNetwork()
    """
    inLayer = LinearLayer(numOfFeatures)
    hiddenLayer0 = SigmoidLayer(numOfHidden)
    #hiddenLayer1 = SigmoidLayer(numOfHidden)
    #hiddenLayer2 = SigmoidLayer(numOfHidden)
    outLayer = LinearLayer(1)
    
    net.addInputModule(inLayer)
    net.addModule(hiddenLayer0)
    #net.addModule(hiddenLayer1)
    #net.addModule(hiddenLayer2)
    net.addOutputModule(outLayer)
    
    in_to_hidden = FullConnection(inLayer, hiddenLayer0)
    #zero2one = FullConnection(hiddenLayer0, hiddenLayer1)
    #one2two = FullConnection(hiddenLayer1, hiddenLayer2)
    hidden_to_out = FullConnection(hiddenLayer0, outLayer)
    
    
    net.addConnection(in_to_hidden)
    #net.addConnection(zero2one)
    #net.addConnection(one2two)
    net.addConnection(hidden_to_out)
    net.sortModules()
    """
    trainer = BackpropTrainer(net, tstdata)
    print 'numOfHidden: ' + str(numOfHidden)
    #res = trainer.trainUntilConvergence()
    for i in range(100):
        res = trainer.train()
    evaluatedData = tstdata
    press0 = []
    press1 = []
    expectedPress0 = []
    expectedPress1 = []
    for input, expectedOutput in evaluatedData:
        output = net.activate(input)
        press0.append(output)
        #press1.append(output[1])
        expectedPress0.append(expectedOutput)
        #expectedPress1.append(expectedOutput[1])
        #errorSum0+=abs(output[0]-expectedOutput[0])
        #errorSum1+=abs(output[1]-expectedOutput[1])
    
    #print errorSum0/len(evaluatedData)
    #print errorSum1/len(evaluatedData)
    print mean_squared_error(press0, expectedPress0)
    print np.mean(expectedPress0)
    #print mean_squared_error(press1, expectedPress1)
    
    """
    arr = np.array(press0)
    print np.std(arr, axis=0)
    arr = np.array(press1)
    print np.std(arr, axis=0)
    print 'end'
    """

#calc()