Python utils.data_iterator函数代码示例

 def predict(self, session, X, y=None):
   """Make predictions from the provided model."""
   # If y is given, the loss is also calculated
   # We deactivate dropout by setting it to 1
   dp = 1
   losses = []
   results = []
   if np.any(y):
       data = data_iterator(X, y, batch_size=self.config.batch_size,
                            label_size=self.config.label_size, shuffle=False)
   else:
       data = data_iterator(X, batch_size=self.config.batch_size,
                            label_size=self.config.label_size, shuffle=False)
   for step, (x, y) in enumerate(data):
     feed = self.create_feed_dict(input_batch=x, dropout=dp)
     if np.any(y):
       feed[self.labels_placeholder] = y
       loss, preds = session.run(
           [self.loss, self.predictions], feed_dict=feed)
       losses.append(loss)
     else:
       preds = session.run(self.predictions, feed_dict=feed)
     predicted_indices = preds.argmax(axis=1)
     results.extend(predicted_indices)
   return np.mean(losses), results

  def run_epoch(self, sess, input_data, input_labels):
    """Runs an epoch of training.

    Trains the model for one-epoch.
  
    Args:
      sess: tf.Session() object
      input_data: np.ndarray of shape (n_samples, n_features)
      input_labels: np.ndarray of shape (n_samples, n_classes)
    Returns:
      average_loss: scalar. Average minibatch loss of model on epoch.
    """
    # And then after everything is built, start the training loop.
    average_loss = 0
    for step, (input_batch, label_batch) in enumerate(
        data_iterator(input_data, input_labels,
                      batch_size=self.config.batch_size,
                      label_size=self.config.n_classes)):

      # Fill a feed dictionary with the actual set of images and labels
      # for this particular training step.
      feed_dict = self.create_feed_dict(input_batch, label_batch)

      # Run one step of the model.  The return values are the activations
      # from the `self.train_op` (which is discarded) and the `loss` Op.  To
      # inspect the values of your Ops or variables, you may include them
      # in the list passed to sess.run() and the value tensors will be
      # returned in the tuple from the call.
      _, loss_value = sess.run([self.train_op, self.loss], feed_dict=feed_dict)
      average_loss += loss_value

    average_loss = average_loss / step
    return average_loss 

 def run_epoch(self, session, input_data, input_labels,
               shuffle=True, verbose=True):
   orig_X, orig_y = input_data, input_labels
   dp = self.config.dropout
   # We're interested in keeping track of the loss and accuracy during training
   total_loss = []
   total_correct_examples = 0
   total_processed_examples = 0
   total_steps = len(orig_X) / self.config.batch_size
   for step, (x, y) in enumerate(
     data_iterator(orig_X, orig_y, batch_size=self.config.batch_size,
                  label_size=self.config.label_size, shuffle=shuffle)):
     feed = self.create_feed_dict(input_batch=x, dropout=dp, label_batch=y)
     loss, total_correct, _ = session.run(
         [self.loss, self.correct_predictions, self.train_op],
         feed_dict=feed)
     total_processed_examples += len(x)
     total_correct_examples += total_correct
     total_loss.append(loss)
     ##
     if verbose and step % verbose == 0:
       sys.stdout.write('\r{} / {} : loss = {}'.format(
           step, total_steps, np.mean(total_loss)))
       sys.stdout.flush()
   if verbose:
       sys.stdout.write('\r')
       sys.stdout.flush()
   return np.mean(total_loss), total_correct_examples / float(total_processed_examples)

 def run_epoch(self, session, sent1_data, sent2_data, len1_data, len2_data, input_labels,
         verbose=100):
     orig_sent1, orig_sent2, orig_len1, orig_len2, orig_y = ( sent1_data,
             sent2_data, len1_data, len2_data, input_labels )
     kp = self.config.kp
     total_loss = []
     total_correct_examples = 0
     total_processed_examples = 0
     total_steps = int( orig_sent1.shape[0] / self.config.batch_size)
     for step, (sent1, sent2, len1, len2, y) in enumerate(
         data_iterator(orig_sent1, orig_sent2, orig_len1, orig_len2, orig_y,
                 batch_size=self.config.batch_size, label_size=self.config.label_size)):
         feed = self.create_feed_dict(sent1, sent2, len1, len2, y, kp)
         loss, total_correct, _ = session.run(
             [self.loss, self.correct_predictions, self.train_op],
             feed_dict=feed)
         total_processed_examples += len(y)
         total_correct_examples += total_correct
         total_loss.append(loss)
         if verbose and step % verbose == 0:
             sys.stdout.write('\r{} / {} : loss = {}'.format(
                 step, total_steps, np.mean(total_loss)))
             sys.stdout.flush()
     if verbose:
         sys.stdout.write('\r')
         sys.stdout.flush()
     return np.mean(total_loss), total_correct_examples / float(total_processed_examples), total_loss

 def run_epoch(self, session, input_data, shuffle=False, verbose=True):
     dp = self.config.dropout
     l2 = self.config.l2
     # We're interested in keeping track of the loss and accuracy during training
     total_loss = []
     total_correct_examples = 0
     total_processed_examples = 0
     if self.config.batch_size == "chapter":
         total_steps = len(input_data)
     else:
         total_steps = len(input_data) / self.config.batch_size
     data = data_iterator(input_data, batch_size=self.config.batch_size, chop_limit=self.config.max_line_length, shuffle=shuffle)
     for step, (lines, line_lengths, labels) in enumerate(data):
         if self.config.weight_loss:
             # f = np.log
             f = lambda x: np.sqrt(x)
             normalization_factor = np.mean([1./f(ct) for ct in self.speakers.speaker_freq.values()])
             self.index_to_weight = {k:1./(normalization_factor*f(self.speakers.speaker_freq[v])) for k,v in self.speakers.index_to_speaker.items()}
             feed = self.create_feed_dict(lines, line_lengths, labels, dp, l2, [self.index_to_weight[l] for l in labels])
         else:
             self.index_to_weight = {k:1. for k in range(len(self.speakers))}
             feed = self.create_feed_dict(lines, line_lengths, labels, dp, l2)
         loss, total_correct, _ = session.run([self.loss, self.correct_predictions, self.train_op], feed_dict=feed)
         total_processed_examples += len(labels)
         total_correct_examples += total_correct
         total_loss.append(loss)
         ##
         if verbose and step % verbose == 0:
             sys.stdout.write('\r{} / {} : loss = {}'.format(step, total_steps, np.mean(total_loss)))
             sys.stdout.flush()
     if verbose:
         sys.stdout.write('\r')
         sys.stdout.flush()
     return np.mean(total_loss), total_correct_examples / float(total_processed_examples)

    def run_epoch(self, session, X, y, trees, shuffle=False, verbose=True):
        loss_history = []
        var_objective_history = []
        total_correct_examples = 0
        total_processed_examples = 0
        total_steps = len(X) / self.batch_size
        for step, (X_batch, y_batch) in enumerate(data_iterator(X, y, batch_size=self.batch_size, shuffle=shuffle)):
            #supervised
            feed = self.create_feed_dict(input_batch=X_batch, label_batch=y_batch)
            loss, total_correct, _ = session.run(
                    [self.loss, self.correct_predictions, self.train_op],
                    feed_dict=feed)
            total_processed_examples += len(X_batch)
            total_correct_examples += total_correct
            loss_history.append(loss)

            #unsupervised
            var_objective, accuracy = self.unsup_update(trees[step])
            var_objective_history.append(var_objective)

            if verbose and step % verbose == 0:
                sys.stdout.write('\r{} / {} : loss = {}, var_objective = {}'.format(
                        step, total_steps, np.mean(loss_history), np.mean(var_objective)))
                sys.stdout.flush()
        if verbose:
                sys.stdout.write('\r')
                sys.stdout.flush()
        return loss_history, total_correct_examples / float(total_processed_examples)

 def predict(self, session, X, y=None):
     losses = []
     results = []
     for step, (x, y) in enumerate(data_iterator(X, y, batch_size=self.batch_size, shuffle=False)):
         feed = self.create_feed_dict(input_batch=x, label_batch=y)
         if np.any(y):
             loss, preds = session.run(
                     [self.loss, self.predictions], feed_dict=feed)
             losses.append(loss)
         else:
             preds = session.run(self.predictions, feed_dict=feed)
         results.extend(preds)
     return results, np.mean(losses)

 def predict(self, session, sent1_data, sent2_data, len1_data, len2_data, y=None):
     """Make predictions from the provided model."""
     # If y is given, the loss is also calculated
     # We deactivate dropout by setting it to 1
     kp = 1.0
     losses = []
     results = []
     if np.any(y):
         data = data_iterator(sent1_data, sent2_data, len1_data, len2_data, y, batch_size=self.config.batch_size,
                              label_size=self.config.label_size, shuffle=False)
     else:
         data = data_iterator(sent1_data, sent2_data, len1_data, len2_data, batch_size=self.config.batch_size,
                              label_size=self.config.label_size, shuffle=False)
     for step, (sent1, sent2, len1, len2, y) in enumerate(data):
         feed = self.create_feed_dict(sent1, sent2, len1, len2, y, kp)
         if np.any(y):
             loss, preds = session.run(
                 [self.loss, self.predictions], feed_dict=feed)
             losses.append(loss)
         else:
             preds = session.run(self.predictions, feed_dict=feed)
         results.extend(preds)
     return np.mean(losses), np.array(results)

 def predict(self, session, input_data, disallow_other=False):
     dp = 1.
     l2 = 0.
     losses = []
     results = []
     data = data_iterator(input_data, batch_size=self.config.batch_size, chop_limit=self.config.max_line_length)
     for step, (lines, line_lengths, labels) in enumerate(data):
         feed = self.create_feed_dict(lines, line_lengths, labels, dp, l2, [self.index_to_weight[l] for l in labels])
         loss, preds, predicted_indices = session.run([self.loss, self.predictions, self.one_hot_predictions], feed_dict=feed)
         if disallow_other:
             preds[:,self.speakers.speaker_to_index["OTHER"]] = 0.
             predicted_indices = preds.argmax(axis=1)
         losses.append(loss)
         results.extend(predicted_indices)
     return np.mean(losses), results

  def run_epoch(self, session, input_data, input_labels,
                shuffle=True, verbose=True):
    orig_X, orig_y = input_data, input_labels
    dp = self.config.dropout
    # We're interested in keeping track of the loss and accuracy during training
    total_loss = []
    total_correct_examples = 0
    total_processed_examples = 0
    total_steps = len(orig_X) / self.config.batch_size
    for step, (x, y) in enumerate(
      data_iterator(orig_X, orig_y, batch_size=self.config.batch_size,
                   label_size=self.config.label_size, shuffle=shuffle)):
      feed = self.create_feed_dict(input_batch=x, dropout=dp, label_batch=y)

      # Add tf.merge_all_summaries() which will provide summaries.
      #merged = tf.merge_all_summaries()
      #writer = tf.train.SummaryWriter("/tmp/q2_NER_logs", session.graph)

      # Run
      #loss, total_correct, _, summary_str = session.run(
    #      [self.loss, self.correct_predictions, self.train_op, merged],
    #      feed_dict=feed)

      loss, total_correct, _ = session.run(
          [self.loss, self.correct_predictions, self.train_op],
          feed_dict=feed)

      #print "========================="
      #print summary_str
      #writer.add_summary(summary_str)

      total_processed_examples += len(x)
      total_correct_examples += total_correct
      total_loss.append(loss)
      ##
      if verbose and step % verbose == 0:
        sys.stdout.write('\r{} / {} : loss = {}'.format(
            step, total_steps, np.mean(total_loss)))
        sys.stdout.flush()
    if verbose:
        sys.stdout.write('\r')
        sys.stdout.flush()
    return np.mean(total_loss), total_correct_examples / float(total_processed_examples)