Python utils_tf.model_eval方法代码示例

# 需要导入模块: from cleverhans import utils_tf [as 别名]
# 或者: from cleverhans.utils_tf import model_eval [as 别名]
def eval_advs(self, x, y, preds_adv, X_test, Y_test, att_type):
        """
        Evaluate the accuracy of the model on adversarial examples

        :param x: symbolic input to model.
        :param y: symbolic variable for the label.
        :param preds_adv: symbolic variable for the prediction on an
                          adversarial example.
        :param X_test: NumPy array of test set inputs.
        :param Y_test: NumPy array of test set labels.
        :param att_type: name of the attack.
        """
        end = (len(X_test) // self.batch_size) * self.batch_size

        if self.hparams.fast_tests:
            end = 10*self.batch_size

        acc = model_eval(self.sess, x, y, preds_adv, X_test[:end],
                         Y_test[:end], args=self.eval_params)
        self.log_value('test_accuracy_%s' % att_type, acc,
                       'Test accuracy on adversarial examples')
        return acc 

# 需要导入模块: from cleverhans import utils_tf [as 别名]
# 或者: from cleverhans.utils_tf import model_eval [as 别名]
def eval_advs(self, x, y, preds_adv, X_test, Y_test, att_type):
    """
    Evaluate the accuracy of the model on adversarial examples

    :param x: symbolic input to model.
    :param y: symbolic variable for the label.
    :param preds_adv: symbolic variable for the prediction on an
                      adversarial example.
    :param X_test: NumPy array of test set inputs.
    :param Y_test: NumPy array of test set labels.
    :param att_type: name of the attack.
    """
    end = (len(X_test) // self.batch_size) * self.batch_size

    if self.hparams.fast_tests:
      end = 10*self.batch_size

    acc = model_eval(self.sess, x, y, preds_adv, X_test[:end],
                     Y_test[:end], args=self.eval_params)
    self.log_value('test_accuracy_%s' % att_type, acc,
                   'Test accuracy on adversarial examples')
    return acc 

# 需要导入模块: from cleverhans import utils_tf [as 别名]
# 或者: from cleverhans.utils_tf import model_eval [as 别名]
def  evaluate():
	eval_params = {'batch_size ': FLAGS.batch_size}
	accuracy = model_eval(sess , x, y, predictions , X_test_scaled , y_test , args= eval_params)
	print('Test  accuracy  on  legitimate  test  examples: ' + str(accuracy)) 

# 需要导入模块: from cleverhans import utils_tf [as 别名]
# 或者: from cleverhans.utils_tf import model_eval [as 别名]
def prep_bbox(sess, x, y, X_train, Y_train, X_test, Y_test,
              nb_epochs, batch_size, learning_rate,
              rng, nb_classes=10, img_rows=28, img_cols=28, nchannels=1):
    """
    Define and train a model that simulates the "remote"
    black-box oracle described in the original paper.
    :param sess: the TF session
    :param x: the input placeholder for MNIST
    :param y: the ouput placeholder for MNIST
    :param X_train: the training data for the oracle
    :param Y_train: the training labels for the oracle
    :param X_test: the testing data for the oracle
    :param Y_test: the testing labels for the oracle
    :param nb_epochs: number of epochs to train model
    :param batch_size: size of training batches
    :param learning_rate: learning rate for training
    :param rng: numpy.random.RandomState
    :return:
    """

    # Define Keras-based TF model graph (for the black-box model)
    nb_filters = 64
    model = cnn_model(nb_filters=nb_filters, nb_classes=nb_classes)

    # Wrap the model in KerasModelWrapper
    model = KerasModelWrapper(model, nb_classes)
    loss = LossCrossEntropy(model, smoothing=0.1)
    predictions = model.get_logits(x)
    print("Defined TensorFlow model graph.")

    # Train an MNIST model
    train_params = {
        'nb_epochs': nb_epochs,
        'batch_size': batch_size,
        'learning_rate': learning_rate
    }
    train(sess, loss, x, y, X_train, Y_train, args=train_params, rng=rng)

    # Print out the accuracy on legitimate data
    eval_params = {'batch_size': batch_size}
    accuracy = model_eval(sess, x, y, predictions, X_test, Y_test,
                          args=eval_params)
    print('Test accuracy of black-box on legitimate test '
          'examples: ' + str(accuracy))

    return model, predictions, accuracy 

# 需要导入模块: from cleverhans import utils_tf [as 别名]
# 或者: from cleverhans.utils_tf import model_eval [as 别名]
def main(argv):
    checkpoint = tf.train.latest_checkpoint(FLAGS.checkpoint_dir)

    if checkpoint is None:
        raise ValueError("Couldn't find latest checkpoint in " +
                         FLAGS.checkpoint_dir)

    train_start = 0
    train_end = 60000
    test_start = 0
    test_end = 10000
    X_train, Y_train, X_test, Y_test = data_mnist(train_start=train_start,
                                                  train_end=train_end,
                                                  test_start=test_start,
                                                  test_end=test_end)

    assert Y_train.shape[1] == 10

    # NOTE: for compatibility with Madry Lab downloadable checkpoints,
    # we cannot enclose this in a scope or do anything else that would
    # change the automatic naming of the variables.
    model = MadryMNIST()

    x_input = tf.placeholder(tf.float32, shape=[None, 784])
    x_image = tf.placeholder(tf.float32, shape=[None, 28, 28, 1])
    y = tf.placeholder(tf.float32, shape=[None, 10])

    if FLAGS.attack_type == 'fgsm':
        fgsm = FastGradientMethod(model)
        fgsm_params = {'eps': 0.3, 'clip_min': 0., 'clip_max': 1.}
        adv_x = fgsm.generate(x_image, **fgsm_params)
    elif FLAGS.attack_type == 'bim':
        bim = BasicIterativeMethod(model)
        bim_params = {'eps': 0.3, 'clip_min': 0., 'clip_max': 1.,
                      'nb_iter': 50,
                      'eps_iter': .01}
        adv_x = bim.generate(x_image, **bim_params)
    else:
        raise ValueError(FLAGS.attack_type)
    preds_adv = model.get_probs(adv_x)

    saver = tf.train.Saver()

    with tf.Session() as sess:
        # Restore the checkpoint
        saver.restore(sess, checkpoint)

        # Evaluate the accuracy of the MNIST model on adversarial examples
        eval_par = {'batch_size': FLAGS.batch_size}
        t1 = time.time()
        acc = model_eval(
            sess, x_image, y, preds_adv, X_test, Y_test, args=eval_par)
        t2 = time.time()
        print("Took", t2 - t1, "seconds")
        print('Test accuracy on adversarial examples: %0.4f\n' % acc) 

# 需要导入模块: from cleverhans import utils_tf [as 别名]
# 或者: from cleverhans.utils_tf import model_eval [as 别名]
def eval_multi(self, inc_epoch=True):
        """
        Run the evaluation on multiple attacks.
        """
        sess = self.sess
        preds = self.preds
        x = self.x_pre
        y = self.y
        X_train = self.X_train
        Y_train = self.Y_train
        X_test = self.X_test
        Y_test = self.Y_test
        writer = self.writer

        self.summary = tf.Summary()
        report = {}

        # Evaluate on train set
        subsample_factor = 100
        X_train_subsampled = X_train[::subsample_factor]
        Y_train_subsampled = Y_train[::subsample_factor]
        acc_train = model_eval(sess, x, y, preds, X_train_subsampled,
                               Y_train_subsampled, args=self.eval_params)
        self.log_value('train_accuracy_subsampled', acc_train,
                       'Clean accuracy, subsampled train')
        report['train'] = acc_train

        # Evaluate on the test set
        acc = model_eval(sess, x, y, preds, X_test, Y_test,
                         args=self.eval_params)
        self.log_value('test_accuracy_natural', acc,
                       'Clean accuracy, natural test')
        report['test'] = acc

        # Evaluate against adversarial attacks
        if self.epoch % self.hparams.eval_iters == 0:
            for att_type in self.attack_type_test:
                adv_x, preds_adv = self.attacks[att_type]
                acc = self.eval_advs(x, y, preds_adv, X_test, Y_test, att_type)
                report[att_type] = acc

        if self.writer:
            writer.add_summary(self.summary, self.epoch)

        # Add examples of adversarial examples to the summary
        if self.writer and self.epoch % 20 == 0 and self.sum_op is not None:
            sm_val = self.sess.run(self.sum_op,
                                   feed_dict={x: X_test[:self.batch_size],
                                              y: Y_test[:self.batch_size]})
            if self.writer:
                writer.add_summary(sm_val)

        self.epoch += 1 if inc_epoch else 0

        return report 

# 需要导入模块: from cleverhans import utils_tf [as 别名]
# 或者: from cleverhans.utils_tf import model_eval [as 别名]
def prep_bbox(sess, x, y, x_train, y_train, x_test, y_test,
              nb_epochs, batch_size, learning_rate,
              rng, nb_classes=10, img_rows=28, img_cols=28, nchannels=1):
    """
    Define and train a model that simulates the "remote"
    black-box oracle described in the original paper.
    :param sess: the TF session
    :param x: the input placeholder for MNIST
    :param y: the ouput placeholder for MNIST
    :param x_train: the training data for the oracle
    :param y_train: the training labels for the oracle
    :param x_test: the testing data for the oracle
    :param y_test: the testing labels for the oracle
    :param nb_epochs: number of epochs to train model
    :param batch_size: size of training batches
    :param learning_rate: learning rate for training
    :param rng: numpy.random.RandomState
    :return:
    """

    # Define TF model graph (for the black-box model)
    nb_filters = 64
    model = ModelBasicCNN('model1', nb_classes, nb_filters)
    loss = CrossEntropy(model, smoothing=0.1)
    predictions = model.get_logits(x)
    print("Defined TensorFlow model graph.")

    # Train an MNIST model
    train_params = {
        'nb_epochs': nb_epochs,
        'batch_size': batch_size,
        'learning_rate': learning_rate
    }
    train(sess, loss, x, y, x_train, y_train, args=train_params, rng=rng)

    # Print out the accuracy on legitimate data
    eval_params = {'batch_size': batch_size}
    accuracy = model_eval(sess, x, y, predictions, x_test, y_test,
                          args=eval_params)
    print('Test accuracy of black-box on legitimate test '
          'examples: ' + str(accuracy))

    return model, predictions, accuracy 

# 需要导入模块: from cleverhans import utils_tf [as 别名]
# 或者: from cleverhans.utils_tf import model_eval [as 别名]
def main(argv):
  checkpoint = tf.train.latest_checkpoint(FLAGS.checkpoint_dir)

  if checkpoint is None:
    raise ValueError("Couldn't find latest checkpoint in " +
                     FLAGS.checkpoint_dir)

  train_start = 0
  train_end = 60000
  test_start = 0
  test_end = 10000
  X_train, Y_train, X_test, Y_test = data_mnist(train_start=train_start,
                                                train_end=train_end,
                                                test_start=test_start,
                                                test_end=test_end)

  assert Y_train.shape[1] == 10

  # NOTE: for compatibility with Madry Lab downloadable checkpoints,
  # we cannot enclose this in a scope or do anything else that would
  # change the automatic naming of the variables.
  model = MadryMNIST()

  x_input = tf.placeholder(tf.float32, shape=[None, 784])
  x_image = tf.placeholder(tf.float32, shape=[None, 28, 28, 1])
  y = tf.placeholder(tf.float32, shape=[None, 10])

  if FLAGS.attack_type == 'fgsm':
    fgsm = FastGradientMethod(model)
    fgsm_params = {'eps': 0.3, 'clip_min': 0., 'clip_max': 1.}
    adv_x = fgsm.generate(x_image, **fgsm_params)
  elif FLAGS.attack_type == 'bim':
    bim = BasicIterativeMethod(model)
    bim_params = {'eps': 0.3, 'clip_min': 0., 'clip_max': 1.,
                  'nb_iter': 50,
                  'eps_iter': .01}
    adv_x = bim.generate(x_image, **bim_params)
  else:
    raise ValueError(FLAGS.attack_type)
  preds_adv = model.get_probs(adv_x)

  saver = tf.train.Saver()

  with tf.Session() as sess:
    # Restore the checkpoint
    saver.restore(sess, checkpoint)

    # Evaluate the accuracy of the MNIST model on adversarial examples
    eval_par = {'batch_size': FLAGS.batch_size}
    t1 = time.time()
    acc = model_eval(
        sess, x_image, y, preds_adv, X_test, Y_test, args=eval_par)
    t2 = time.time()
    print("Took", t2 - t1, "seconds")
    print('Test accuracy on adversarial examples: %0.4f\n' % acc) 

# 需要导入模块: from cleverhans import utils_tf [as 别名]
# 或者: from cleverhans.utils_tf import model_eval [as 别名]
def eval_multi(self, inc_epoch=True):
    """
    Run the evaluation on multiple attacks.
    """
    sess = self.sess
    preds = self.preds
    x = self.x_pre
    y = self.y
    X_train = self.X_train
    Y_train = self.Y_train
    X_test = self.X_test
    Y_test = self.Y_test
    writer = self.writer

    self.summary = tf.Summary()
    report = {}

    # Evaluate on train set
    subsample_factor = 100
    X_train_subsampled = X_train[::subsample_factor]
    Y_train_subsampled = Y_train[::subsample_factor]
    acc_train = model_eval(sess, x, y, preds, X_train_subsampled,
                           Y_train_subsampled, args=self.eval_params)
    self.log_value('train_accuracy_subsampled', acc_train,
                   'Clean accuracy, subsampled train')
    report['train'] = acc_train

    # Evaluate on the test set
    acc = model_eval(sess, x, y, preds, X_test, Y_test,
                     args=self.eval_params)
    self.log_value('test_accuracy_natural', acc,
                   'Clean accuracy, natural test')
    report['test'] = acc

    # Evaluate against adversarial attacks
    if self.epoch % self.hparams.eval_iters == 0:
      for att_type in self.attack_type_test:
        _, preds_adv = self.attacks[att_type]
        acc = self.eval_advs(x, y, preds_adv, X_test, Y_test, att_type)
        report[att_type] = acc

    if self.writer:
      writer.add_summary(self.summary, self.epoch)

    # Add examples of adversarial examples to the summary
    if self.writer and self.epoch % 20 == 0 and self.sum_op is not None:
      sm_val = self.sess.run(self.sum_op,
                             feed_dict={x: X_test[:self.batch_size],
                                        y: Y_test[:self.batch_size]})
      if self.writer:
        writer.add_summary(sm_val)

    self.epoch += 1 if inc_epoch else 0

    return report 

# 需要导入模块: from cleverhans import utils_tf [as 别名]
# 或者: from cleverhans.utils_tf import model_eval [as 别名]
def prep_bbox(sess, x, y, x_train, y_train, x_test, y_test,
              nb_epochs, batch_size, learning_rate,
              rng, nb_classes=10, img_rows=28, img_cols=28, nchannels=1):
  """
  Define and train a model that simulates the "remote"
  black-box oracle described in the original paper.
  :param sess: the TF session
  :param x: the input placeholder for MNIST
  :param y: the ouput placeholder for MNIST
  :param x_train: the training data for the oracle
  :param y_train: the training labels for the oracle
  :param x_test: the testing data for the oracle
  :param y_test: the testing labels for the oracle
  :param nb_epochs: number of epochs to train model
  :param batch_size: size of training batches
  :param learning_rate: learning rate for training
  :param rng: numpy.random.RandomState
  :return:
  """

  # Define TF model graph (for the black-box model)
  nb_filters = 64
  model = ModelBasicCNN('model1', nb_classes, nb_filters)
  loss = CrossEntropy(model, smoothing=0.1)
  predictions = model.get_logits(x)
  print("Defined TensorFlow model graph.")

  # Train an MNIST model
  train_params = {
      'nb_epochs': nb_epochs,
      'batch_size': batch_size,
      'learning_rate': learning_rate
  }
  train(sess, loss, x_train, y_train, args=train_params, rng=rng)

  # Print out the accuracy on legitimate data
  eval_params = {'batch_size': batch_size}
  accuracy = model_eval(sess, x, y, predictions, x_test, y_test,
                        args=eval_params)
  print('Test accuracy of black-box on legitimate test '
        'examples: ' + str(accuracy))

  return model, predictions, accuracy