Python mnist.MNIST类代码示例

	def __init__(self):
		#Load MNIST datset
		mnistData = MNIST('./mnistData')
		self.imgTrain,self.lblTrain=mnistData.load_training()
		#self.imgTrainSmpl=self.imgTrain[:50000]
		self.imgTrainSmpl = [[2.5,2.4],[0.5,0.7],[2.2,2.9],[1.9,2.2],[3.1,3.0],[2.3,2.7],[2,1.6],[1,1.1],[1.5,1.6],[1.1,0.9]]
		np.seterr(all='warn')

def importMNIST(folder,resolution,classes,amount,signals):
    print 'importing MNIST data...'
    if os.path.isfile('saved_DY.pkl'):
        print 'found file'
        f = open('saved_DY.pkl','r')
        D = pickle.load(f)
        D_labels = pickle.load(f)
        Y = pickle.load(f)
        Y_labels = pickle.load(f)

        return np.matrix(D),D_labels,np.matrix(Y),Y_labels

    mndata = MNIST(folder)
    train_ims,train_labels = mndata.load_training()
    print 'training loaded'
    test_ims,test_labels = mndata.load_testing()
    print 'testing loaded'

    training_samples = resize(np.array(train_ims),resolution)
    training_labels = np.array(train_labels)
    D,D_labels = organize(training_samples,training_labels,classes,amount)
    print 'dictionary, D, made'

    random_idx = np.array(np.random.permutation(10000))[0:signals] #10000 is total signals avail

    Y = (resize(np.array(test_ims),resolution))[:,random_idx]
    Y_labels = np.array(test_labels)[random_idx]
    print 'signals, Y, made'

    saveToFile(D,D_labels,Y,Y_labels)

    return np.matrix(D),D_labels,np.matrix(Y),Y_labels

    def __init__(self,collect_gold_standard):
        n_neighbors = 15

        mndata = MNIST('/home/ggdhines/Databases/mnist')
        training = mndata.load_training()

        digits = range(0,10)

        training_dict = {d:[] for d in digits}

        for t_index,label in enumerate(training[1]):
            training_dict[label].append(training[0][t_index])

        weight = "distance"
        self.clf = neighbors.KNeighborsClassifier(n_neighbors, weights=weight)

        pca = PCA(n_components=50)
        self.T = pca.fit(training[0])
        reduced_training = self.T.transform(training[0])
        # print sum(pca.explained_variance_ratio_)
        # clf.fit(training[0], training[1])
        self.clf.fit(reduced_training, training[1])

        self.transcribed_digits = {d:[] for d in digits}
        self.collect_gold_standard = collect_gold_standard

        self.cells_to_process = []
        self.completed_cells = []

def load_data(dataset="mnist_small", center=False):
    '''
        @param dataset: The dataset to load
        @param random_state: random state to control random parameter

        Load a specified dataset currently only
        "mnist_small" and "mnist" are supported
    '''
    if (dataset == "mnist_small"):
        X_train = np.loadtxt("./mldata/mnist_small/X_train", delimiter=",").reshape(1540,64)
        X_test = np.loadtxt("./mldata/mnist_small/X_test", delimiter=",").reshape(257,64)
        y_train = np.loadtxt("./mldata/mnist_small/y_train", delimiter=",")
        y_test = np.loadtxt("./mldata/mnist_small/y_test", delimiter=",")
        X_train = X_train[:,:,np.newaxis]
        X_test = X_test[:,:,np.newaxis]
    elif dataset == "mnist":
        mndata = MNIST('./mldata/mnist')
        X_train, y_train = map(np.array, mndata.load_training())
        X_test, y_test = map(np.array, mndata.load_testing())
        X_train = X_train/255.0
        X_test = X_test/255.0
        X_train = X_train[:,:,np.newaxis]
        X_test = X_test[:,:,np.newaxis]
    elif dataset == "cifar":
        (X_train, y_train), (X_test, y_test) = load_cifar()

    else:
        raise Exception("Datset not found")

    return (X_train, y_train), (X_test, y_test)

def load_dataset():
    mndata = MNIST("./data/")
    X_train, labels_train = map(np.array, mndata.load_training())
    X_test, labels_test = map(np.array, mndata.load_testing())
    X_train = X_train / 255.0
    X_test = X_test / 255.0
    X_train = X_train[:, :, np.newaxis]
    X_test = X_test[:, :, np.newaxis]
    return (X_train, labels_train), (X_test, labels_test)

    def __init__(self):
        Classifier.__init__(self)

        self.classifier = svm.SVC(gamma=0.001,probability=True)

        mndata = MNIST('/home/ggdhines/Databases/mnist')
        training = mndata.load_training()

        self.classifier.fit(training[0], training[1])

 def load_data(self):
     mn = MNIST('.')
     mn.test()
     data = mn.train_images
     data = np.array(data)
     
     data.astype(np.float32)
     data = data/255.0
     return data

    def load(self,pixels_per_cell = (8,8),cells_per_block=(3,3),orientations=9):
        '''
        Generates a Data Set

        Parameters: None

        Returns:    train_set     - Training Set of 10000 images
                    train_labels  - Training Set Labels of corresponding images
                    test_set      - Test Set of 10000 images
                    test_labels   - Test Set Labels of corresponding images
        '''
        mn = MNIST('./data')
        train_raw = mn.load_training()
        test_raw = mn.load_testing()

        print "Loaded Raw images"

        learning_set = []
        Boom = {}
        for i in range(10):
            Boom[str(i)] = []
        for i in range(0,60000):
            Boom[str(train_raw[1][i])].append(train_raw[0][i])
        for i in range(0,10000):
            Boom[str(test_raw[1][i])].append(test_raw[0][i])
        t = datetime.now().microsecond
        random.seed(t)
        [random.shuffle(Boom[str(i)]) for i in range(10)]

        print "Choosing 20000 training images uniformly randomly"

        # Descriptor Generator
        for l in range(10):
            for i in range(0,2000):
                img =  np.array(Boom[str(l)][i])
                img.shape = (28,28)
                fd, hog_image = hog(img, orientations=orientations, pixels_per_cell=pixels_per_cell,cells_per_block=cells_per_block, visualise=True)
                learning_set.append([fd,l])

        print "Data Points now chosen and Generated HOG descriptors for them"

        t = datetime.now().microsecond
        random.seed(t)
        print "Shuffling Chosen Data Set"
        random.shuffle(learning_set)

        for i in range(20000):
            self.learning_set.append(learning_set[i][0])
            self.learning_set_labels.append(learning_set[i][1])

        print "Data Loading and Distribution Succesfully done"

        self.train_set = self.learning_set[:10000]
        self.train_labels = self.learning_set_labels[:10000]
        self.test_set = self.learning_set[10000:20000]
        self.test_labels = self.learning_set_labels[10000:20000]

def load_data(datadir, is_training=False):

    mn = MNIST(datadir)

    if is_training:
        img, label = mn.load_training()
    else:
        img, label = mn.load_testing()

    return img, label

def loaddata():
    #Loading mnist data using python-mnist library
    mnLoader = MNIST('asgndata/mnist')
    data1 = mnLoader.load_training() # train data
    data2 = mnLoader.load_testing()  # test data

    features = np.array(data1[0]+data2[0], 'int16')
    labels = np.array(data1[1]+data2[1], 'int')
    X_train, y_train, X_test, y_test = preprocessData(features, labels)
    return X_train, y_train, X_test, y_test 

 def load_targets(self):
     mn = MNIST('.')
     mn.test()
     targets = []
     for t in mn.train_labels:
         #print t
         out = np.zeros(self.output)
         out[t] = 1
         targets.append(out)
     targets = np.array(targets)
     return targets

 def __init__(self,k):
     #Define k value
     self.k=k
     #Load MNIST datset
     mnistData=MNIST('./mnistData')
     self.imgTrain,self.lblTrain=mnistData.load_training()
     self.imgTest,self.lblTest=mnistData.load_testing()
     #Initialize the random centroids
     self.imgCen=[]
     for c in xrange(self.k):
         self.imgCen.append([random.randint(0,255) for d in xrange(784)])

def train_rls():
    mndata = MNIST("./data")
    X_train, Y_train = mndata.load_training()
    X_test, Y_test = mndata.load_testing()
    X_train, X_test = np.array(X_train), np.array(X_test)
    #One-vs-all mapping
    Y_train = ova(Y_train)
    Y_test = ova(Y_test)
    #Train greedy RLS, select 50 features
    cb = Callback(X_test, Y_test)
    learner = GreedyRLS(X_train, Y_train, 50, callbackfun=cb)
    print("Selected features " +str(learner.selected))

def run():
  TorchModel = PyTorchHelpers.load_lua_class('torch_model.lua', 'TorchModel')
  torchModel = TorchModel(backend, 28, 10)

  mndata = MNIST('../../data/mnist')
  imagesList, labelsList = mndata.load_training()
  labels = np.array(labelsList, dtype=np.uint8)
  images = np.array(imagesList, dtype=np.float32)
  labels += 1  # since torch/lua labels are 1-based
  N = labels.shape[0]
  print('loaded mnist training data')

  if numTrain > 0:
    N = min(N, numTrain)
  print('numExamples N', N)
  numBatches = N // batchSize
  for epoch in range(numEpochs):
    epochLoss = 0
    epochNumRight = 0
    for b in range(numBatches):
      res = torchModel.trainBatch(
        learningRate,
        images[b * batchSize:(b+1) * batchSize],
        labels[b * batchSize:(b+1) * batchSize])
#      print('res', res)
      numRight = res['numRight']
      loss = res['loss']
      epochNumRight += numRight
      epochLoss += loss
      print('epoch ' + str(epoch) + ' batch ' + str(b) + ' accuracy: ' + str(numRight * 100.0 / batchSize) + '%')
    print('epoch ' + str(epoch) + ' accuracy: ' + str(epochNumRight * 100.0 / N) + '%')

  print('finished training')
  print('loading test data...')
  imagesList, labelsList = mndata.load_testing()
  labels = np.array(labelsList, dtype=np.uint8)
  images = np.array(imagesList, dtype=np.float32)
  labels += 1  # since torch/lua labels are 1-based
  N = labels.shape[0]
  print('loaded mnist testing data')

  numBatches = N // batchSize
  epochLoss = 0
  epochNumRight = 0
  for b in range(numBatches):
    predictions = torchModel.predict(images[b * batchSize:(b+1) * batchSize]).asNumpyTensor().reshape(batchSize)
    labelsBatch = labels[b * batchSize:(b+1) * batchSize]
    numRight = (predictions == labelsBatch).sum()
    epochNumRight += numRight
  print('test results: accuracy: ' + str(epochNumRight * 100.0 / N) + '%')

def run(args):

    nn_args = {}

    if args.output_activation:
        activation_class = getattr(activation_functions, args.output_activation)
        nn_args['output_activation'] = activation_class()

    nn = Network(args.shape, seed=42, **nn_args)

    print "Loading the training data"
    mndata = MNIST(args.training_data)
    training_data, training_labels = mndata.load_training()

    training_data = convert_training_data(training_data)
    training_labels = convert_number_labels_to_vectors(training_labels)

    fitted, epochs = nn.SGD(training_data, training_labels,
                            epochs=args.epochs,
                            mini_batch_size=args.mini_batch_size,
                            eta=args.eta,
                            save_history=args.save_epochs)

    if args.testing_data:
        print "Testing data"
        test_data, test_labels = mndata.load_testing()
        test_data = convert_training_data(test_data)
        # For evaluation, we put the index of the label
        # with the argmax
        evaluation = fitted.evaluate(test_data, test_labels,
                                     evaluator=np.argmax)
        print evaluation

    if args.save:

        label_dir = mkdir_or_temp(args.save)

        fitted_path = "{}/nn.pkl".format(label_dir)

        with open(fitted_path, 'wb') as handle:
            pickle.dump(fitted, handle)

        if epochs is not None:
            for i, epoch in enumerate(epochs):
                epoch_path = '{}/nn_epoch_{}.pkl'.format(label_dir, i)
                with open(epoch_path, 'wb') as handle:
                    pickle.dump(epoch, handle)
                    print "Saved epoch {} to {}".format(i, epoch_path)