Python MNIST.load_testing方法代码示例

# 需要导入模块: from mnist import MNIST [as 别名]
# 或者: from mnist.MNIST import load_testing [as 别名]
def load_data(dataset="mnist_small", center=False):
    '''
        @param dataset: The dataset to load
        @param random_state: random state to control random parameter

        Load a specified dataset currently only
        "mnist_small" and "mnist" are supported
    '''
    if (dataset == "mnist_small"):
        X_train = np.loadtxt("./mldata/mnist_small/X_train", delimiter=",").reshape(1540,64)
        X_test = np.loadtxt("./mldata/mnist_small/X_test", delimiter=",").reshape(257,64)
        y_train = np.loadtxt("./mldata/mnist_small/y_train", delimiter=",")
        y_test = np.loadtxt("./mldata/mnist_small/y_test", delimiter=",")
        X_train = X_train[:,:,np.newaxis]
        X_test = X_test[:,:,np.newaxis]
    elif dataset == "mnist":
        mndata = MNIST('./mldata/mnist')
        X_train, y_train = map(np.array, mndata.load_training())
        X_test, y_test = map(np.array, mndata.load_testing())
        X_train = X_train/255.0
        X_test = X_test/255.0
        X_train = X_train[:,:,np.newaxis]
        X_test = X_test[:,:,np.newaxis]
    elif dataset == "cifar":
        (X_train, y_train), (X_test, y_test) = load_cifar()

    else:
        raise Exception("Datset not found")

    return (X_train, y_train), (X_test, y_test)

# 需要导入模块: from mnist import MNIST [as 别名]
# 或者: from mnist.MNIST import load_testing [as 别名]
def importMNIST(folder,resolution,classes,amount,signals):
    print 'importing MNIST data...'
    if os.path.isfile('saved_DY.pkl'):
        print 'found file'
        f = open('saved_DY.pkl','r')
        D = pickle.load(f)
        D_labels = pickle.load(f)
        Y = pickle.load(f)
        Y_labels = pickle.load(f)

        return np.matrix(D),D_labels,np.matrix(Y),Y_labels

    mndata = MNIST(folder)
    train_ims,train_labels = mndata.load_training()
    print 'training loaded'
    test_ims,test_labels = mndata.load_testing()
    print 'testing loaded'

    training_samples = resize(np.array(train_ims),resolution)
    training_labels = np.array(train_labels)
    D,D_labels = organize(training_samples,training_labels,classes,amount)
    print 'dictionary, D, made'

    random_idx = np.array(np.random.permutation(10000))[0:signals] #10000 is total signals avail

    Y = (resize(np.array(test_ims),resolution))[:,random_idx]
    Y_labels = np.array(test_labels)[random_idx]
    print 'signals, Y, made'

    saveToFile(D,D_labels,Y,Y_labels)

    return np.matrix(D),D_labels,np.matrix(Y),Y_labels

# 需要导入模块: from mnist import MNIST [as 别名]
# 或者: from mnist.MNIST import load_testing [as 别名]
def load_dataset():
    mndata = MNIST("./data/")
    X_train, labels_train = map(np.array, mndata.load_training())
    X_test, labels_test = map(np.array, mndata.load_testing())
    X_train = X_train / 255.0
    X_test = X_test / 255.0
    X_train = X_train[:, :, np.newaxis]
    X_test = X_test[:, :, np.newaxis]
    return (X_train, labels_train), (X_test, labels_test)

# 需要导入模块: from mnist import MNIST [as 别名]
# 或者: from mnist.MNIST import load_testing [as 别名]
    def load(self,pixels_per_cell = (8,8),cells_per_block=(3,3),orientations=9):
        '''
        Generates a Data Set

        Parameters: None

        Returns:    train_set     - Training Set of 10000 images
                    train_labels  - Training Set Labels of corresponding images
                    test_set      - Test Set of 10000 images
                    test_labels   - Test Set Labels of corresponding images
        '''
        mn = MNIST('./data')
        train_raw = mn.load_training()
        test_raw = mn.load_testing()

        print "Loaded Raw images"

        learning_set = []
        Boom = {}
        for i in range(10):
            Boom[str(i)] = []
        for i in range(0,60000):
            Boom[str(train_raw[1][i])].append(train_raw[0][i])
        for i in range(0,10000):
            Boom[str(test_raw[1][i])].append(test_raw[0][i])
        t = datetime.now().microsecond
        random.seed(t)
        [random.shuffle(Boom[str(i)]) for i in range(10)]

        print "Choosing 20000 training images uniformly randomly"

        # Descriptor Generator
        for l in range(10):
            for i in range(0,2000):
                img =  np.array(Boom[str(l)][i])
                img.shape = (28,28)
                fd, hog_image = hog(img, orientations=orientations, pixels_per_cell=pixels_per_cell,cells_per_block=cells_per_block, visualise=True)
                learning_set.append([fd,l])

        print "Data Points now chosen and Generated HOG descriptors for them"

        t = datetime.now().microsecond
        random.seed(t)
        print "Shuffling Chosen Data Set"
        random.shuffle(learning_set)

        for i in range(20000):
            self.learning_set.append(learning_set[i][0])
            self.learning_set_labels.append(learning_set[i][1])

        print "Data Loading and Distribution Succesfully done"

        self.train_set = self.learning_set[:10000]
        self.train_labels = self.learning_set_labels[:10000]
        self.test_set = self.learning_set[10000:20000]
        self.test_labels = self.learning_set_labels[10000:20000]

# 需要导入模块: from mnist import MNIST [as 别名]
# 或者: from mnist.MNIST import load_testing [as 别名]
def load_data(datadir, is_training=False):

    mn = MNIST(datadir)

    if is_training:
        img, label = mn.load_training()
    else:
        img, label = mn.load_testing()

    return img, label

# 需要导入模块: from mnist import MNIST [as 别名]
# 或者: from mnist.MNIST import load_testing [as 别名]
def loaddata():
    #Loading mnist data using python-mnist library
    mnLoader = MNIST('asgndata/mnist')
    data1 = mnLoader.load_training() # train data
    data2 = mnLoader.load_testing()  # test data

    features = np.array(data1[0]+data2[0], 'int16')
    labels = np.array(data1[1]+data2[1], 'int')
    X_train, y_train, X_test, y_test = preprocessData(features, labels)
    return X_train, y_train, X_test, y_test 

# 需要导入模块: from mnist import MNIST [as 别名]
# 或者: from mnist.MNIST import load_testing [as 别名]
 def __init__(self,k):
     #Define k value
     self.k=k
     #Load MNIST datset
     mnistData=MNIST('./mnistData')
     self.imgTrain,self.lblTrain=mnistData.load_training()
     self.imgTest,self.lblTest=mnistData.load_testing()
     #Initialize the random centroids
     self.imgCen=[]
     for c in xrange(self.k):
         self.imgCen.append([random.randint(0,255) for d in xrange(784)])

# 需要导入模块: from mnist import MNIST [as 别名]
# 或者: from mnist.MNIST import load_testing [as 别名]
def train_rls():
    mndata = MNIST("./data")
    X_train, Y_train = mndata.load_training()
    X_test, Y_test = mndata.load_testing()
    X_train, X_test = np.array(X_train), np.array(X_test)
    #One-vs-all mapping
    Y_train = ova(Y_train)
    Y_test = ova(Y_test)
    #Train greedy RLS, select 50 features
    cb = Callback(X_test, Y_test)
    learner = GreedyRLS(X_train, Y_train, 50, callbackfun=cb)
    print("Selected features " +str(learner.selected))

# 需要导入模块: from mnist import MNIST [as 别名]
# 或者: from mnist.MNIST import load_testing [as 别名]
def run():
  TorchModel = PyTorchHelpers.load_lua_class('torch_model.lua', 'TorchModel')
  torchModel = TorchModel(backend, 28, 10)

  mndata = MNIST('../../data/mnist')
  imagesList, labelsList = mndata.load_training()
  labels = np.array(labelsList, dtype=np.uint8)
  images = np.array(imagesList, dtype=np.float32)
  labels += 1  # since torch/lua labels are 1-based
  N = labels.shape[0]
  print('loaded mnist training data')

  if numTrain > 0:
    N = min(N, numTrain)
  print('numExamples N', N)
  numBatches = N // batchSize
  for epoch in range(numEpochs):
    epochLoss = 0
    epochNumRight = 0
    for b in range(numBatches):
      res = torchModel.trainBatch(
        learningRate,
        images[b * batchSize:(b+1) * batchSize],
        labels[b * batchSize:(b+1) * batchSize])
#      print('res', res)
      numRight = res['numRight']
      loss = res['loss']
      epochNumRight += numRight
      epochLoss += loss
      print('epoch ' + str(epoch) + ' batch ' + str(b) + ' accuracy: ' + str(numRight * 100.0 / batchSize) + '%')
    print('epoch ' + str(epoch) + ' accuracy: ' + str(epochNumRight * 100.0 / N) + '%')

  print('finished training')
  print('loading test data...')
  imagesList, labelsList = mndata.load_testing()
  labels = np.array(labelsList, dtype=np.uint8)
  images = np.array(imagesList, dtype=np.float32)
  labels += 1  # since torch/lua labels are 1-based
  N = labels.shape[0]
  print('loaded mnist testing data')

  numBatches = N // batchSize
  epochLoss = 0
  epochNumRight = 0
  for b in range(numBatches):
    predictions = torchModel.predict(images[b * batchSize:(b+1) * batchSize]).asNumpyTensor().reshape(batchSize)
    labelsBatch = labels[b * batchSize:(b+1) * batchSize]
    numRight = (predictions == labelsBatch).sum()
    epochNumRight += numRight
  print('test results: accuracy: ' + str(epochNumRight * 100.0 / N) + '%')

# 需要导入模块: from mnist import MNIST [as 别名]
# 或者: from mnist.MNIST import load_testing [as 别名]
def run(args):

    nn_args = {}

    if args.output_activation:
        activation_class = getattr(activation_functions, args.output_activation)
        nn_args['output_activation'] = activation_class()

    nn = Network(args.shape, seed=42, **nn_args)

    print "Loading the training data"
    mndata = MNIST(args.training_data)
    training_data, training_labels = mndata.load_training()

    training_data = convert_training_data(training_data)
    training_labels = convert_number_labels_to_vectors(training_labels)

    fitted, epochs = nn.SGD(training_data, training_labels,
                            epochs=args.epochs,
                            mini_batch_size=args.mini_batch_size,
                            eta=args.eta,
                            save_history=args.save_epochs)

    if args.testing_data:
        print "Testing data"
        test_data, test_labels = mndata.load_testing()
        test_data = convert_training_data(test_data)
        # For evaluation, we put the index of the label
        # with the argmax
        evaluation = fitted.evaluate(test_data, test_labels,
                                     evaluator=np.argmax)
        print evaluation

    if args.save:

        label_dir = mkdir_or_temp(args.save)

        fitted_path = "{}/nn.pkl".format(label_dir)

        with open(fitted_path, 'wb') as handle:
            pickle.dump(fitted, handle)

        if epochs is not None:
            for i, epoch in enumerate(epochs):
                epoch_path = '{}/nn_epoch_{}.pkl'.format(label_dir, i)
                with open(epoch_path, 'wb') as handle:
                    pickle.dump(epoch, handle)
                    print "Saved epoch {} to {}".format(i, epoch_path)

# 需要导入模块: from mnist import MNIST [as 别名]
# 或者: from mnist.MNIST import load_testing [as 别名]
def main(kernel):
  print "Loading the data"

  mn = MNIST(DATA_PATH)

  test_img, test_label = mn.load_testing()
  train_img, train_label = mn.load_training()
  
  train_img = np.array(train_img[:SIZE_TRAIN])
  train_label = np.array(train_label[:SIZE_TRAIN])
  test_img = np.array(test_img[:SIZE_TEST])
  test_label = np.array(test_label[:SIZE_TEST])

  print "Finished loading the data"

  # Create a classifier: a support vector classifier
  if kernel == 'rbf':
    print "Training with RBF kernel - Might take a few minutes"
    classifier = svm.SVC(C=10, gamma=5e-7, kernel='rbf') 
  elif kernel == 'linear':
    print "Training with Linear kernel - Might take a few minutes"
    classifier = svm.SVC(C=1e-6, kernel='linear')
  elif kernel == 'poly':
    print "Training with Polynomial kernel - Might take a few minutes"
    #classifier = svm.SVC(C=10, gamma=1e-7, kernel='poly', degree=2)
    #classifier = svm.SVC(C=10, gamma=1e-6, kernel='poly', degree=3)
    classifier = svm.SVC(C=10, gamma=1e-6, kernel='poly', degree=4)

  # We learn the digits on the first half of the digits
  classifier.fit(train_img, train_label)

  print "Classifying - Might take a few minutes"

  predicted = classifier.predict(test_img)
  print predicted

  cm = metrics.confusion_matrix(test_label, predicted)

  print("Classification report for classifier %s:\n%s\n"% (classifier, metrics.classification_report(test_label, predicted)))
  print("Confusion matrix:\n%s" % cm)

  cm_normalized = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]

  plt.figure()
  plot_confusion_matrix(cm_normalized, title='Normalized confusion matrix')

  print "Result: %s"%(np.trace(cm_normalized)/10)

# 需要导入模块: from mnist import MNIST [as 别名]
# 或者: from mnist.MNIST import load_testing [as 别名]
def main():
    data = MNIST('./data')

    def transform(x):
        return x / 255.

    # 60,000 train samples of 28x28 grid, domain 0-255
    mnist_train_data, mnist_train_label = data.load_training()
    mnist_train_data_norm = np.array([transform(np.array(x)) for x in mnist_train_data])

    mlp_config = {'hidden_layer_sizes': (1000,),
                  'activation': 'relu',
                  'algorithm': 'adam',
                  'max_iter': 20,
                  'early_stopping': True,
                  'validation_fraction': 0.1,
                  'verbose': True
                  }
    mnist_classifier = nn.MLPClassifier(**mlp_config)
    mnist_classifier.fit(X=mnist_train_data_norm, y=mnist_train_label)

    # 10,000 test samples
    mnist_test_data, mnist_test_label = data.load_testing()
    mnist_test_data_norm = np.array([transform(np.array(x)) for x in mnist_test_data])

    prediction = mnist_classifier.predict_proba(mnist_test_data_norm)
    truth_array = [prediction[idx].argmax() == mnist_test_label[idx] for idx in range(len(prediction))]
    accuracy = float(sum(truth_array)) / float(len(truth_array))
    print "out of sample model accuracy [%s]" % accuracy

    print "serializing to pmml without transform (User defined transform not yet supported"
    pmml_path = "./model_pmml"
    if not os.path.exists(pmml_path):
        os.mkdir(pmml_path)
    sklearn2pmml(mnist_classifier, None, pmml_path + "/MLP_MNIST.pmml", with_repr=True)

    print "serializing with joblib for import in python"
    # KJS TODO: Serialize transform with the model
    pickle_path = "./model_pickle"
    if not os.path.exists(pickle_path):
        os.mkdir(pickle_path)
    joblib.dump(mnist_classifier, pickle_path + "/MLP_MNIST.pkl")

# 需要导入模块: from mnist import MNIST [as 别名]
# 或者: from mnist.MNIST import load_testing [as 别名]
def load_mnist():
	download_mnist()
	mnist_data = MNIST(MNIST_PATH)
	train_imgs, train_labels = mnist_data.load_training()
	test_imgs, test_labels = mnist_data.load_testing()
	data = {}

	for i in range(len(train_imgs)):
		square = []
		for j in range(1024):
			row = j / 32
			col = j % 32
			if (row < 2 or col < 2 or row > 29 or col > 29):
				square.append(0)
			else:
				val = train_imgs[i][(row - 2) * 28 + col - 2]
				square.append(val)
		train_imgs[i] = square

	for i in range(len(test_imgs)):
		square = []
		for j in range(1024):
			row = j / 32
			col = j % 32
			if (row < 2 or col < 2 or row > 29 or col > 29):
				square.append(0)
			else:
				val = test_imgs[i][(row - 2) * 28 + col - 2]
				square.append(val)
		test_imgs[i] = square

	data["train_imgs"] = np.array(train_imgs, dtype="f").reshape(60000, 1, 32, 32)
	data["test_imgs"] = np.array(test_imgs, dtype="f").reshape(10000, 1, 32, 32)
	data["train_labels"] = np.array(train_labels)
	data["test_labels"] = np.array(test_labels)

	preprocess(data["train_imgs"], data["test_imgs"])

	data["train_no"] = 60000
	data["test_no"] = 10000

	return data

# 需要导入模块: from mnist import MNIST [as 别名]
# 或者: from mnist.MNIST import load_testing [as 别名]
def main():
    data = MNIST('./data')
    col_names = ["x" + str(x) for x in range(784)]
    # Define a transform function that will be serialized with the model
    mnist_mapper = sklearn_pandas.DataFrameMapper([(col_names, StandardScaler()), ("digit", None)])

    # 60,000 train samples of 28x28 grid, domain 0-255
    mnist_train_data, mnist_train_label = data.load_training()
    mnist_train_df = pandas.concat((pandas.DataFrame(mnist_train_data, columns=col_names),
                                    pandas.DataFrame(list(mnist_train_label), columns=["digit"])),
                                   axis=1)
    mnist_train_df_norm = mnist_mapper.fit_transform(mnist_train_df)

    mlp_config = {'hidden_layer_sizes': (1000,),
                  'activation': 'tanh',
                  'algorithm': 'adam',
                  'max_iter': 20,
                  'early_stopping': True,
                  'validation_fraction': 0.1,
                  'verbose': True
                  }
    mnist_classifier = nn.MLPClassifier(**mlp_config)
    mnist_classifier.fit(X=mnist_train_df_norm[:, 0:28 * 28], y=mnist_train_df_norm[:, 28 * 28])

    # 10,000 test samples
    mnist_test_data, mnist_test_label = data.load_testing()
    mnist_test_df = pandas.concat((pandas.DataFrame(mnist_test_data, columns=col_names),
                                   pandas.DataFrame(list(mnist_test_label), columns=["digit"])),
                                  axis=1)
    mnist_test_df_norm = mnist_mapper.fit_transform(mnist_test_df)

    prediction = mnist_classifier.predict_proba(mnist_test_df_norm[:, 0:28 * 28])
    truth_array = [prediction[idx].argmax() == mnist_test_label[idx] for idx in range(len(prediction))]
    accuracy = float(sum(truth_array)) / float(len(truth_array))
    print "out of sample model accuracy [%s]" % accuracy
    print "serializing to pmml"
    sklearn2pmml(mnist_classifier, mnist_mapper, "MLP_MNIST.pmml", with_repr=True)

# 需要导入模块: from mnist import MNIST [as 别名]
# 或者: from mnist.MNIST import load_testing [as 别名]
from mnist import MNIST
import numpy as np
import matplotlib.pyplot as plt
import pickle as cPickle

white = 0.001

mndata = MNIST('mnist/') #directiry holding the MNIST dataset

mndata.load_training()
mndata.load_testing()

train_data = np.reshape(mndata.train_images, (60000, 28, 28))
test_data = np.reshape(mndata.test_images, (10000, 28, 28))

train_data = train_data / 255.
test_data = test_data / 255.

for x in range(0,len(train_data)):
	train_data[x] = np.rot90(train_data[x], 3)

for x in range(0,len(test_data)):
	test_data[x] = np.rot90(test_data[x], 3)

trn_labels = list()
tst_labels = list()

white_space=[0 for p in range(11)]
white_space[10] = 1

t = 2