Python tensorflow.RandomShuffleQueue方法代码示例

# 需要导入模块: import tensorflow [as 别名]
# 或者: from tensorflow import RandomShuffleQueue [as 别名]
def _setup_base_graph(self):
        """
        Set up queue, variables and session
        """
        self.graph = tf.Graph()
        with self.graph.as_default() as g:
            input_dim = self.input_dim
            batch_size = self.batch_size
            hidden_units = self.hidden_units
            layer_units = [input_dim] + hidden_units + [1]
            layer_num = len(layer_units)

            #make Queue for getting batch
            self.queue = q = tf.RandomShuffleQueue(capacity=self.q_capacity,
                                        min_after_dequeue=self.min_after_dequeue,
                                        dtypes=["float", "float"],
                                        shapes=[[input_dim], [input_dim]])
            #input data
            self.data1, self.data2 = q.dequeue_many(batch_size, name="inputs")

            self._setup_variables()
            self._setup_training()
            self._setup_prediction()
            self._setup_pretraining() 

# 需要导入模块: import tensorflow [as 别名]
# 或者: from tensorflow import RandomShuffleQueue [as 别名]
def get_multitrack_input(shape, batch_size, name="", input_shape=None):
    '''
    Creates multitrack placeholders and a random shuffle queue based on it
    :param input_shape: Shape of accompaniment and voice magnitudes
    :param batch_size: Number of samples in each batch
    :param name: How to name the placeholders
    :return: [List of mixture,acc,voice placeholders, random shuffle queue, symbolic batch sample from queue]
    '''
    m,a,v = get_multitrack_placeholders(shape, input_shape=input_shape)

    min_after_dequeue = 0
    buffer = 1000
    capacity = min_after_dequeue + buffer

    if input_shape is None:
        input_shape = shape
    queue = tf.RandomShuffleQueue(capacity, min_after_dequeue, [tf.float32, tf.float32, tf.float32], [input_shape, shape, shape])
    input_batch = queue.dequeue_many(batch_size, name="input_batch" + name)

    return [m,a,v], queue, input_batch 

# 需要导入模块: import tensorflow [as 别名]
# 或者: from tensorflow import RandomShuffleQueue [as 别名]
def build_pipeline(self, name):
        """Creates a new input subgraph composed of the following components:
            - Reader queue that feeds protobuf data files.
            - RandomShuffleQueue assigned parallel-thread queuerunners.
            - Dynamic padded-bucketed-batching queue for organizing batches in a time and
              space-efficient manner.

        Args:
            name: filename prefix for data. See Dataset class for naming conventions.

        Returns:
            2-tuple (lengths, sequences):
                lengths: (dict) parsed context feature from protobuf file.
                Supports keys in LENGTHS.
                sequences: (dict) parsed feature_list from protobuf file.
                Supports keys in SEQUENCES.
        """
        with tf.variable_scope(name + '_pipeline'):
            proto_text = self._read_line(self.paths[name + '_tfrecords'])
            context_pair, sequence_pair = self._assign_queue(proto_text)
            input_length = tf.add(context_pair['encoder_sequence_length'],
                                  context_pair['decoder_sequence_length'],
                                  name=name + 'length_add')
            return self._padded_bucket_batches(input_length, sequence_pair) 

# 需要导入模块: import tensorflow [as 别名]
# 或者: from tensorflow import RandomShuffleQueue [as 别名]
def _assign_queue(self, proto_text):
        """
        Args:
            proto_text: object to be enqueued and managed by parallel threads.
        """

        with tf.variable_scope('shuffle_queue'):
            queue = tf.RandomShuffleQueue(
                capacity=self.capacity,
                min_after_dequeue=10*self.batch_size,
                dtypes=tf.string, shapes=[()])

            enqueue_op = queue.enqueue(proto_text)
            example_dq = queue.dequeue()

            qr = tf.train.QueueRunner(queue, [enqueue_op] * 4)
            tf.train.add_queue_runner(qr)

            _sequence_lengths, _sequences = tf.parse_single_sequence_example(
                serialized=example_dq,
                context_features=LENGTHS,
                sequence_features=SEQUENCES)
        return _sequence_lengths, _sequences 

# 需要导入模块: import tensorflow [as 别名]
# 或者: from tensorflow import RandomShuffleQueue [as 别名]
def testScalarShapes(self):
    with self.test_session() as sess:
      q = tf.RandomShuffleQueue(
          10, 0, [tf.int32, tf.int32],
          shapes=[(), (1,)])
      q.enqueue_many([[1, 2, 3, 4], [[5], [6], [7], [8]]]).run()
      q.enqueue([9, [10]]).run()
      dequeue_t = q.dequeue()
      results = []
      for _ in range(2):
        a, b = sess.run(dequeue_t)
        results.append((a, b))
      a, b = sess.run(q.dequeue_many(3))
      for i in range(3):
        results.append((a[i], b[i]))
      self.assertItemsEqual([(1, [5]), (2, [6]), (3, [7]), (4, [8]), (9, [10])],
                            results) 

# 需要导入模块: import tensorflow [as 别名]
# 或者: from tensorflow import RandomShuffleQueue [as 别名]
def testParallelEnqueue(self):
    with self.test_session() as sess:
      q = tf.RandomShuffleQueue(10, 0, tf.float32)
      elems = [10.0, 20.0, 30.0, 40.0, 50.0, 60.0, 70.0, 80.0, 90.0, 100.0]
      enqueue_ops = [q.enqueue((x,)) for x in elems]
      dequeued_t = q.dequeue()

      # Run one producer thread for each element in elems.
      def enqueue(enqueue_op):
        sess.run(enqueue_op)
      threads = [self.checkedThread(target=enqueue, args=(e,))
                 for e in enqueue_ops]
      for thread in threads:
        thread.start()
      for thread in threads:
        thread.join()

      # Dequeue every element using a single thread.
      results = []
      for _ in xrange(len(elems)):
        results.append(dequeued_t.eval())
      self.assertItemsEqual(elems, results) 

# 需要导入模块: import tensorflow [as 别名]
# 或者: from tensorflow import RandomShuffleQueue [as 别名]
def testMultiEnqueueAndDequeue(self):
    with self.test_session() as sess:
      q = tf.RandomShuffleQueue(
          10, 0, (tf.int32, tf.float32))
      elems = [(5, 10.0), (10, 20.0), (15, 30.0)]
      enqueue_ops = [q.enqueue((x, y)) for x, y in elems]
      dequeued_t = q.dequeue()

      for enqueue_op in enqueue_ops:
        enqueue_op.run()

      results = []
      for _ in xrange(len(elems)):
        x, y = sess.run(dequeued_t)
        results.append((x, y))
      self.assertItemsEqual(elems, results) 

# 需要导入模块: import tensorflow [as 别名]
# 或者: from tensorflow import RandomShuffleQueue [as 别名]
def testEmptyDequeueManyWithNoShape(self):
    with self.test_session():
      q = tf.RandomShuffleQueue(10, 0, tf.float32)
      enqueue_op = q.enqueue(
          (tf.constant([10.0, 20.0], shape=(1, 2)),))
      dequeued_t = q.dequeue_many(0)

      # Expect the operation to fail due to the shape not being constrained.
      with self.assertRaisesOpError(
          "require the components to have specified shapes"):
        dequeued_t.eval()

      enqueue_op.run()

      # RandomShuffleQueue does not make any attempt to support DequeueMany
      # with unspecified shapes, even if a shape could be inferred from the
      # elements enqueued.
      with self.assertRaisesOpError(
          "require the components to have specified shapes"):
        dequeued_t.eval() 

# 需要导入模块: import tensorflow [as 别名]
# 或者: from tensorflow import RandomShuffleQueue [as 别名]
def testEmptyDequeueUpToWithNoShape(self):
    with self.test_session():
      q = tf.RandomShuffleQueue(10, 0, tf.float32)
      enqueue_op = q.enqueue(
          (tf.constant([10.0, 20.0], shape=(1, 2)),))
      dequeued_t = q.dequeue_up_to(0)

      # Expect the operation to fail due to the shape not being constrained.
      with self.assertRaisesOpError(
          "require the components to have specified shapes"):
        dequeued_t.eval()

      enqueue_op.run()

      # RandomShuffleQueue does not make any attempt to support DequeueUpTo
      # with unspecified shapes, even if a shape could be inferred from the
      # elements enqueued.
      with self.assertRaisesOpError(
          "require the components to have specified shapes"):
        dequeued_t.eval() 

# 需要导入模块: import tensorflow [as 别名]
# 或者: from tensorflow import RandomShuffleQueue [as 别名]
def testParallelDequeueMany(self):
    with self.test_session() as sess:
      q = tf.RandomShuffleQueue(1000, 0, tf.float32, shapes=())
      elems = [10.0 * x for x in range(1000)]
      enqueue_op = q.enqueue_many((elems,))
      dequeued_t = q.dequeue_many(100)

      enqueue_op.run()

      # Dequeue 100 items in parallel on 10 threads.
      dequeued_elems = []

      def dequeue():
        dequeued_elems.extend(sess.run(dequeued_t))
      threads = [self.checkedThread(target=dequeue) for _ in range(10)]
      for thread in threads:
        thread.start()
      for thread in threads:
        thread.join()
      self.assertItemsEqual(elems, dequeued_elems) 

# 需要导入模块: import tensorflow [as 别名]
# 或者: from tensorflow import RandomShuffleQueue [as 别名]
def testParallelDequeueUpTo(self):
    with self.test_session() as sess:
      q = tf.RandomShuffleQueue(1000, 0, tf.float32, shapes=())
      elems = [10.0 * x for x in range(1000)]
      enqueue_op = q.enqueue_many((elems,))
      dequeued_t = q.dequeue_up_to(100)

      enqueue_op.run()

      # Dequeue 100 items in parallel on 10 threads.
      dequeued_elems = []

      def dequeue():
        dequeued_elems.extend(sess.run(dequeued_t))
      threads = [self.checkedThread(target=dequeue) for _ in range(10)]
      for thread in threads:
        thread.start()
      for thread in threads:
        thread.join()
      self.assertItemsEqual(elems, dequeued_elems) 

# 需要导入模块: import tensorflow [as 别名]
# 或者: from tensorflow import RandomShuffleQueue [as 别名]
def testParallelDequeueUpToRandomPartition(self):
    with self.test_session() as sess:
      dequeue_sizes = [random.randint(50, 150) for _ in xrange(10)]
      total_elements = sum(dequeue_sizes)
      q = tf.RandomShuffleQueue(total_elements, 0, tf.float32, shapes=())

      elems = [10.0 * x for x in xrange(total_elements)]
      enqueue_op = q.enqueue_many((elems,))
      dequeue_ops = [q.dequeue_up_to(size) for size in dequeue_sizes]

      enqueue_op.run()

      # Dequeue random number of items in parallel on 10 threads.
      dequeued_elems = []

      def dequeue(dequeue_op):
        dequeued_elems.extend(sess.run(dequeue_op))
      threads = []
      for dequeue_op in dequeue_ops:
        threads.append(self.checkedThread(target=dequeue, args=(dequeue_op,)))
      for thread in threads:
        thread.start()
      for thread in threads:
        thread.join()
      self.assertItemsEqual(elems, dequeued_elems) 

# 需要导入模块: import tensorflow [as 别名]
# 或者: from tensorflow import RandomShuffleQueue [as 别名]
def testDequeueUpToWithTensorParameter(self):
    with self.test_session():
      # Define a first queue that contains integer counts.
      dequeue_counts = [random.randint(1, 10) for _ in range(100)]
      count_q = tf.RandomShuffleQueue(100, 0, tf.int32)
      enqueue_counts_op = count_q.enqueue_many((dequeue_counts,))
      total_count = sum(dequeue_counts)

      # Define a second queue that contains total_count elements.
      elems = [random.randint(0, 100) for _ in range(total_count)]
      q = tf.RandomShuffleQueue(
          total_count, 0, tf.int32, ((),))
      enqueue_elems_op = q.enqueue_many((elems,))

      # Define a subgraph that first dequeues a count, then DequeuesUpTo
      # that number of elements.
      dequeued_t = q.dequeue_up_to(count_q.dequeue())

      enqueue_counts_op.run()
      enqueue_elems_op.run()

      dequeued_elems = []
      for _ in dequeue_counts:
        dequeued_elems.extend(dequeued_t.eval())
      self.assertItemsEqual(elems, dequeued_elems) 

# 需要导入模块: import tensorflow [as 别名]
# 或者: from tensorflow import RandomShuffleQueue [as 别名]
def testDequeueFromClosedQueue(self):
    with self.test_session():
      q = tf.RandomShuffleQueue(10, 2, tf.float32)
      elems = [10.0, 20.0, 30.0, 40.0]
      enqueue_op = q.enqueue_many((elems,))
      close_op = q.close()
      dequeued_t = q.dequeue()

      enqueue_op.run()
      close_op.run()
      results = [dequeued_t.eval() for _ in elems]
      expected = [[elem] for elem in elems]
      self.assertItemsEqual(expected, results)

      # Expect the operation to fail due to the queue being closed.
      with self.assertRaisesRegexp(tf.errors.OutOfRangeError,
                                   "is closed and has insufficient"):
        dequeued_t.eval() 

# 需要导入模块: import tensorflow [as 别名]
# 或者: from tensorflow import RandomShuffleQueue [as 别名]
def testBlockingDequeueFromClosedEmptyQueue(self):
    with self.test_session() as sess:
      q = tf.RandomShuffleQueue(10, 0, tf.float32)
      close_op = q.close()
      dequeued_t = q.dequeue()

      finished = []  # Needs to be a mutable type
      def dequeue():
        # Expect the operation to fail due to the queue being closed.
        with self.assertRaisesRegexp(tf.errors.OutOfRangeError,
                                     "is closed and has insufficient"):
          sess.run(dequeued_t)
        finished.append(True)

      dequeue_thread = self.checkedThread(target=dequeue)
      dequeue_thread.start()
      # The close_op should run after the dequeue_thread has blocked.
      # TODO(mrry): Figure out how to do this without sleeping.
      time.sleep(0.1)
      self.assertEqual(len(finished), 0)
      close_op.run()
      dequeue_thread.join()
      self.assertEqual(len(finished), 1)