Python input_data.read_data_sets函数代码示例

def BuildComplexLearner(restore=True):
    """Builds a Complex Learner that uses CNNs for digit classification.
    
    Args:
        restore: (bool) Whether to restore the model or train a new one.
    """
    if restore:
        mnist = input_data.read_data_sets("MNIST_data/", one_hot=True)
        learner = ComplexLearner()
        learner.Restore("thresholded_model.ckpt")
        return learner
    else:
        mnist = input_data.read_data_sets("MNIST_data/", one_hot=True)
        learner = ComplexLearner()
        ThresholdPixels(mnist.train.images)

        def signal_handler(signal, frame):
            print "Caught ctrl-c. Saving model then exiting..."
            learner.Save("thresholded_ctrl_c.ckpt")
            sys.exit(0)

        signal.signal(signal.SIGINT, signal_handler)
        learner.Train(mnist.train)
        ThresholdPixels(mnist.test.images)
        learner.Test(mnist.test)
        return learner

def fetch_data():
    if not exists(data_dir):
        makedirs(data_dir)

    # Normalize data once if we haven't done it before and store it in a file
    if not exists(f'{data_dir}/{data_file}'):
        print('Downloading MNIST data')
        mnist = input_data.read_data_sets(data_dir, one_hot=True)

        def _normalize(data, mean=None, std=None):
            if mean is None:
                mean = np.mean(data, axis=0)
                std = np.std(data, axis=0)
            return div0((data - mean), std), mean, std

        train_data, mean, std = _normalize(mnist.train.images)

        validation_data, *_ = _normalize(mnist.validation.images, mean, std)
        test_data, *_ = _normalize(mnist.test.images, mean, std)

        mnist_data = {'train_images': train_data,
                      'train_labels': mnist.train.labels,
                      'validation_images': validation_data,
                      'validation_labels': mnist.validation.labels,
                      'test_images': test_data,
                      'test_labels': mnist.test.labels}
        with open(f'{data_dir}/{data_file}', 'wb') as f:
            pickle.dump(mnist_data, f)

    # If we have normalized the data already; load it
    else:
        with open(f'{data_dir}/{data_file}', 'rb') as f:
            mnist_data = pickle.load(f)

    return mnist_data

def load_data(data_dir):
  """Returns training and test tf.data.Dataset objects."""
  data = input_data.read_data_sets(data_dir, one_hot=True)
  train_ds = tf.data.Dataset.from_tensor_slices((data.train.images,
                                                 data.train.labels))
  test_ds = tf.data.Dataset.from_tensors((data.test.images, data.test.labels))
  return (train_ds, test_ds)

def load_data(name, random_labels=False):
	"""Load the data
	name - the name of the dataset
	random_labels - True if we want to return random labels to the dataset
	return object with data and labels"""
	print ('Loading Data...')
	C = type('type_C', (object,), {})
	data_sets = C()
	if name.split('/')[-1] == 'MNIST':
		data_sets_temp = input_data.read_data_sets(os.path.dirname(sys.argv[0]) + "/data/MNIST_data/", one_hot=True)
		data_sets.data = np.concatenate((data_sets_temp.train.images, data_sets_temp.test.images), axis=0)
		data_sets.labels = np.concatenate((data_sets_temp.train.labels, data_sets_temp.test.labels), axis=0)
	else:
		d = sio.loadmat(os.path.join(os.path.dirname(sys.argv[0]), name + '.mat'))
		F = d['F']
		y = d['y']
		C = type('type_C', (object,), {})
		data_sets = C()
		data_sets.data = F
		data_sets.labels = np.squeeze(np.concatenate((y[None, :], 1 - y[None, :]), axis=0).T)
	# If we want to assign random labels to the  data
	if random_labels:
		labels = np.zeros(data_sets.labels.shape)
		labels_index = np.random.randint(low=0, high=labels.shape[1], size=labels.shape[0])
		labels[np.arange(len(labels)), labels_index] = 1
		data_sets.labels = labels
	return data_sets

def main(_):
  # Import data
  mnist = input_data.read_data_sets(FLAGS.data_dir, one_hot=True)
  # Create the model
  x = tf.placeholder(tf.float32, [None, 784])
  W = tf.Variable(tf.zeros([784, 10]))
  b = tf.Variable(tf.zeros([10]))
  y = tf.matmul(x, W) + b
  # Define loss and optimizer
  y_ = tf.placeholder(tf.float32, [None, 10])
  # The raw formulation of cross-entropy,
  #
  #   tf.reduce_mean(-tf.reduce_sum(y_ * tf.log(tf.nn.softmax(y)),
  #                                 reduction_indices=[1]))
  #
  # can be numerically unstable.
  #
  # So here we use tf.nn.softmax_cross_entropy_with_logits on the raw
  # outputs of 'y', and then average across the batch.
  cross_entropy = tf.reduce_mean(
      tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y))
  train_step = tf.train.GradientDescentOptimizer(0.5).minimize(cross_entropy)
  sess = tf.InteractiveSession()
  tf.global_variables_initializer().run()
  # Train
  for _ in range(1000):
    batch_xs, batch_ys = mnist.train.next_batch(100)
    sess.run(train_step, feed_dict={x: batch_xs, y_: batch_ys})
  # Test trained model
  correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))
  accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
  print(sess.run(accuracy, feed_dict={x: mnist.test.images,
                                      y_: mnist.test.labels}))

def main(_):

  # Create a cluster from the parameter server and worker hosts.
  cluster = tf.train.ClusterSpec({"local": ["localhost:2222"]})
  
  # Create and start a server for the local task.
  server = tf.train.Server(cluster, job_name="local", task_index=0)

  # Build model...
  from tensorflow.examples.tutorials.mnist import input_data
  mnist = input_data.read_data_sets("MNIST_data/", one_hot=True)
  x = tf.placeholder(tf.float32, [None, 784])
  W = tf.Variable(tf.zeros([784, 10]))
  b = tf.Variable(tf.zeros([10]))
  y = tf.nn.softmax(tf.matmul(x, W) + b)
  y_ = tf.placeholder(tf.float32, [None, 10])
  cross_entropy = tf.reduce_mean(-tf.reduce_sum(y_ * tf.log(y), reduction_indices=[1]))
  train_step = tf.train.GradientDescentOptimizer(0.5).minimize(cross_entropy)
  init = tf.initialize_all_variables()
  sess = tf.Session(server.target)
  sess.run(init)
  for i in range(1000):
    batch_xs, batch_ys = mnist.train.next_batch(100)
    sess.run(train_step, feed_dict={x: batch_xs, y_: batch_ys})

  correct_prediction = tf.equal(tf.argmax(y,1), tf.argmax(y_,1))
  accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
  print(sess.run(accuracy, feed_dict={x: mnist.test.images, y_: mnist.test.labels}))

def main(_):
  cluster,server,job_name,task_index,num_workers = get_mpi_cluster_server_jobname(num_ps = 4, num_workers = 5)
  MY_GPU = task_index % NUM_GPUS

  if job_name == "ps":
    server.join()
  elif job_name == "worker":

    is_chief = (task_index == 0)
    # Assigns ops to the local worker by default.
    with tf.device(tf.train.replica_device_setter(\
      worker_device='/job:worker/task:{}/gpu:{}'.format(task_index,MY_GPU),
		  cluster=cluster)):

      loss,accuracy,input_tensor,true_output_tensor = get_loss_accuracy_ops()

      global_step = tf.Variable(0,trainable=False)
      optimizer = tf.train.AdagradOptimizer(0.01)
      if sync_mode:
        optimizer = tf.train.SyncReplicasOptimizer(optimizer,replicas_to_aggregate=num_workers,
          replica_id=task_index,total_num_replicas=num_workers)

      train_op = optimizer.minimize(loss, global_step=global_step)

      if sync_mode and is_chief:
        # Initial token and chief queue runners required by the sync_replicas mode
        chief_queue_runner = optimizer.get_chief_queue_runner()
        init_tokens_op = optimizer.get_init_tokens_op()

      saver = tf.train.Saver()
      summary_op = tf.merge_all_summaries()
      init_op = tf.initialize_all_variables()

    # Create a "supervisor", which oversees the training process.
    sv = tf.train.Supervisor(is_chief=is_chief,logdir="/tmp/train_logs",init_op=init_op,summary_op=summary_op,
                             saver=saver,global_step=global_step,save_model_secs=600)

    mnist = input_data.read_data_sets(data_dir, one_hot=True)

    # The supervisor takes care of session initialization, restoring from
    # a checkpoint, and closing when done or an error occurs.
    config = tf.ConfigProto(allow_soft_placement=True)
    with sv.prepare_or_wait_for_session(server.target,config=config) as sess:
      if sync_mode and is_chief:
        sv.start_queue_runners(sess,[chief_queue_runner])
        sess.run(init_tokens_op)

      step = 0
      start = time.time()
      while not sv.should_stop() and step < 1000:
        batch_xs, batch_ys = mnist.train.next_batch(batch_size)
        train_feed = {input_tensor: batch_xs, true_output_tensor: batch_ys,K.learning_phase(): 1}

        _, step, curr_loss, curr_accuracy = sess.run([train_op, global_step, loss, accuracy], feed_dict=train_feed)
      	sys.stdout.write('\rWorker {}, step: {}, loss: {}, accuracy: {}'.format(task_index,step,curr_loss,curr_accuracy))
      	sys.stdout.flush()

    # Ask for all the services to stop.
    sv.stop()
    print('Elapsed: {}'.format(time.time() - start))

    def test_fully_connected(self):
#         self.mock.verbose = True
        self.mock.loop_cnt = 65
        self.mock.add_layer(784)
        self.mock.add_cnn()
        self.mock.add_pool()
        self.mock.add_cnn()
        self.mock.add_pool()
        self.mock.add_layer(1024, act_func=tf.nn.relu)
        self.mock.add_layer(10, act_func= tf.nn.softmax)
        self.mock.set_entropy_func(self.mock.entropy_log)
        
        from tensorflow.examples.tutorials.mnist import input_data
        mnist = input_data.read_data_sets("down/", one_hot=True)

        def get_feed(x=None):
            b = mnist.train.next_batch(100)
            feed = {self.mock.input:b[0], self.mock.target:b[1]}
            return feed
            
        self.mock.get_feed_before_loop = get_feed
        self.mock.get_feed_each_one_step = get_feed
        
#         def print_entropy (i, sess, feed):
#             print  sess.run( self.mock.entropy , feed)
#         self.mock.after_one_step = print_entropy
            
        self.mock.learn()
        self.assertTrue(0.5 <self.mock.last_acc , 'less 0.5 acc %2.3f'%self.mock.last_acc )

    def __init__(self, config, sess):
        self.input_dim = config.input_dim      # 784
        self.z_dim = config.z_dim              # 14
        self.c_cat = config.c_cat              # 10: Category c - 1 hot vector for 10 label values
        self.c_cont = config.c_cont            # 2: Continuous c
        self.d_update = config.d_update        # 2: Run discriminator twice before generator
        self.batch_size = config.batch_size
        self.nepoch = config.nepoch
        self.lr = config.lr                    # Learning rate 0.001
        self.max_grad_norm = config.max_grad_norm  # 40
        self.show_progress = config.show_progress  # False

        self.optimizer = tf.train.AdamOptimizer

        self.checkpoint_dir = config.checkpoint_dir
        self.image_dir = config.image_dir

        home = str(Path.home())
        DATA_ROOT_DIR = os.path.join(home, "dataset", "MNIST_data")
        self.mnist = input_data.read_data_sets(DATA_ROOT_DIR, one_hot=True)

        self.random_seed = 42

        self.X = tf.placeholder(tf.float32, [None, self.input_dim], 'X')
        self.z = tf.placeholder(tf.float32, [None, self.z_dim], 'z')
        self.c_i = tf.placeholder(tf.float32, [None, self.c_cat], 'c_cat')
        self.c_j = tf.placeholder(tf.float32, [None, self.c_cont], 'c_cont')
        self.c = tf.concat([self.c_i, self.c_j], axis=1)
        self.z_c = tf.concat([self.z, self.c_i, self.c_j], axis=1)

        self.training = tf.placeholder_with_default(False, shape=(), name='training')

        self.sess = sess

def main(_):
  # MNIST 데이터 셋을 ont-hot 인코딩 형태로 받아온다.
  mnist = input_data.read_data_sets(FLAGS.data_dir, one_hot=True)

  # Create the model
  x = tf.placeholder(tf.float32, [None, 784])
  W = tf.Variable(tf.zeros([784, 10]))
  b = tf.Variable(tf.zeros([10]))
  y = tf.matmul(x, W) + b

  # Define loss and optimizer
  y_ = tf.placeholder(tf.float32, [None, 10])

  cross_entropy = tf.reduce_mean(
      tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y))
  train_step = tf.train.GradientDescentOptimizer(0.5).minimize(cross_entropy)
  #train_step = tf.train.AdamOptimizer(0.1).minimize(cross_entropy)

  sess = tf.InteractiveSession()
  tf.global_variables_initializer().run()
  # Train
  for _ in range(10000):
    batch_xs, batch_ys = mnist.train.next_batch(100)
    sess.run(train_step, feed_dict={x: batch_xs, y_: batch_ys})

  # Test trained model
  correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))
  accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
  print(sess.run(accuracy, feed_dict={x: mnist.test.images,
                                      y_: mnist.test.labels}))

def get_data(task_name):
    ## Data sets
    if task_name == 'qianli_func':
        (X_train, Y_train, X_cv, Y_cv, X_test, Y_test) = get_data_from_file(file_name='./f_1d_cos_no_noise_data.npz')
    elif task_name == 'f_2D_task2':
        (X_train, Y_train, X_cv, Y_cv, X_test, Y_test) = get_data_from_file(file_name='./f_2d_task2_ml_data_and_mesh.npz')
    elif task_name == 'f_2d_task2_xsinglog1_x_depth2':
        (X_train, Y_train, X_cv, Y_cv, X_test, Y_test) = get_data_from_file(file_name='./f_2d_task2_ml_xsinlog1_x_depth_2data_and_mesh.npz')
    elif task_name == 'f_2d_task2_xsinglog1_x_depth3':
        (X_train, Y_train, X_cv, Y_cv, X_test, Y_test) = get_data_from_file(file_name='./f_2d_task2_ml_xsinlog1_x_depth_3data_and_mesh.npz')
    elif task_name == 'MNIST_flat':
        mnist = input_data.read_data_sets("MNIST_data/", one_hot=True)
        X_train, Y_train = mnist.train.images, mnist.train.labels
        X_cv, Y_cv = mnist.validation.images, mnist.validation.labels
        X_test, Y_test = mnist.test.images, mnist.test.labels
    elif task_name == 'hrushikesh':
        with open('../hrushikesh/patient_data_X_Y.json', 'r') as f_json:
            patients_data = json.load(f_json)
        X = patients_data['1']['X']
        Y = patients_data['1']['Y']
        X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.40)
        X_cv, X_test, Y_cv, Y_test = train_test_split(X_test, Y_test, test_size=0.5)
        (X_train, Y_train, X_cv, Y_cv, X_test, Y_test) = ( np.array(X_train), np.array(Y_train), np.array(X_cv), np.array(Y_cv), np.array(X_test), np.array(Y_test) )
    else:
        raise ValueError('task_name: %s does not exist. Try experiment that exists'%(task_name))
    return (X_train, Y_train, X_cv, Y_cv, X_test, Y_test)

def main():
    data_path = '/home/charlesxu/Workspace/data/MNIST_data/'
    data = input_data.read_data_sets(data_path, one_hot=True)

    original(data)
    widen(data)
    deepen(data)

    def __init__(self, batch_size):
        from tensorflow.examples.tutorials.mnist import input_data
        self.mnist = input_data.read_data_sets('MNIST_data', one_hot=True)

        self.x = tf.placeholder(tf.float32, shape=[batch_size, 28, 28, 1])
        self.feed_y = tf.placeholder(tf.float32, shape=[batch_size, 10])
        self.y = ((2*self.feed_y)-1)

def main(_):
  n_in = 784
  n_out = 10
  n_hidden = 200
  mnist = input_data.read_data_sets(FLAGS.data_dir, one_hot=True)
  w_in = tf.Variable(tf.random_normal([n_in, n_hidden]))
  b_in = tf.Variable(tf.random_normal([n_hidden]))
  w_out = tf.Variable(tf.random_normal([n_hidden, n_out]))
  b_out = tf.Variable(tf.random_normal([n_out]))
  # Create the model
  x = tf.placeholder(tf.float32, [None, n_in])
  h = tf.nn.relu(tf.add(tf.matmul(x, w_in), b_in))
  y = tf.add(tf.matmul(h, w_out), b_out)

  batch_size = 100
  labels = tf.placeholder(tf.float32, [None, n_out])
  cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(y, labels))
  optimizer = tf.train.AdamOptimizer(0.01).minimize(cost)
  with tf.Session() as sess:
    # Train
    sess.run(tf.initialize_all_variables())
    for _ in range(5000):
      batch_xs, batch_ys = mnist.train.next_batch(batch_size)
      sess.run(optimizer, feed_dict={x: batch_xs, labels: batch_ys})
      #print(sess.run(tf.nn.softmax(y), feed_dict={x: batch_xs}))

    # Test trained model
    correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(labels, 1))
    accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
    print(sess.run(accuracy, feed_dict={x: mnist.test.images,
                                        labels: mnist.test.labels}))

def runMNIST():
    imageSize = 4
    imageChannels = 1

    mnist = input_data.read_data_sets('MNIST_data', one_hot=True)

    self.createNetwork("fullyConnected", imageSize)