Python tensorflow.PaddingFIFOQueue方法代码示例

# 需要导入模块: import tensorflow [as 别名]
# 或者: from tensorflow import PaddingFIFOQueue [as 别名]
def get_input_queues(path, word2idx, batch_size=32, num_threads=8):
    input_ph = tf.placeholder(tf.int32, shape=[None])  # [T]
    queue = tf.PaddingFIFOQueue(shapes=[[None, ]], dtypes=[tf.int32], capacity=5000,)

    # TODO: enqueue_many would be faster, would require batch and padding at numpy-level
    enqueue_op = queue.enqueue([input_ph])
    def enqueue_data(sess):
        # for epoch in range(epoch_size):
        while True:  # 
            for idx, line in enumerate(read_data(path)):
                v = vectorize(line, word2idx)
                sess.run(enqueue_op, feed_dict={input_ph: v})

    # dequeue_batch = queue.dequeue_many(batch_size)
    dequeue_op = queue.dequeue()
    dequeue_batch = tf.train.batch([dequeue_op], batch_size=batch_size, num_threads=num_threads, capacity=1000, 
        dynamic_pad=True, name="batch_and_pad")

    # TODO: get corpus_size here
    return enqueue_data, dequeue_batch 

# 需要导入模块: import tensorflow [as 别名]
# 或者: from tensorflow import PaddingFIFOQueue [as 别名]
def testMultiQueueConstructor(self):
    with tf.Graph().as_default():
      q = tf.PaddingFIFOQueue(5, (tf.int32, tf.float32),
                              ((), ()),
                              shared_name="foo", name="Q")
    self.assertTrue(isinstance(q.queue_ref, tf.Tensor))
    self.assertEquals(tf.string_ref, q.queue_ref.dtype)
    self.assertProtoEquals("""
      name:'Q' op:'PaddingFIFOQueue'
      attr { key: 'component_types' value { list {
        type: DT_INT32 type : DT_FLOAT
      } } }
      attr { key: 'shapes' value { list { shape { } shape { } } } }
      attr { key: 'capacity' value { i: 5 } }
      attr { key: 'container' value { s: '' } }
      attr { key: 'shared_name' value { s: 'foo' } }
      """, q.queue_ref.op.node_def) 

# 需要导入模块: import tensorflow [as 别名]
# 或者: from tensorflow import PaddingFIFOQueue [as 别名]
def testConstructorWithShapes(self):
    with tf.Graph().as_default():
      q = tf.PaddingFIFOQueue(5, (tf.int32, tf.float32),
                              shapes=(tf.TensorShape([1, 1, 2, 3]),
                                      tf.TensorShape([5, 8])), name="Q")
    self.assertTrue(isinstance(q.queue_ref, tf.Tensor))
    self.assertEquals(tf.string_ref, q.queue_ref.dtype)
    self.assertProtoEquals("""
      name:'Q' op:'PaddingFIFOQueue'
      attr { key: 'component_types' value { list {
        type: DT_INT32 type : DT_FLOAT
      } } }
      attr { key: 'shapes' value { list {
        shape { dim { size: 1 }
                dim { size: 1 }
                dim { size: 2 }
                dim { size: 3 } }
        shape { dim { size: 5 }
                dim { size: 8 } }
      } } }
      attr { key: 'capacity' value { i: 5 } }
      attr { key: 'container' value { s: '' } }
      attr { key: 'shared_name' value { s: '' } }
      """, q.queue_ref.op.node_def) 

# 需要导入模块: import tensorflow [as 别名]
# 或者: from tensorflow import PaddingFIFOQueue [as 别名]
def testParallelEnqueue(self):
    with self.test_session() as sess:
      q = tf.PaddingFIFOQueue(10, tf.float32, ((),))
      elems = [10.0, 20.0, 30.0, 40.0, 50.0, 60.0, 70.0, 80.0, 90.0, 100.0]
      enqueue_ops = [q.enqueue((x,)) for x in elems]
      dequeued_t = q.dequeue()

      # Run one producer thread for each element in elems.
      def enqueue(enqueue_op):
        sess.run(enqueue_op)
      threads = [self.checkedThread(target=enqueue, args=(e,))
                 for e in enqueue_ops]
      for thread in threads:
        thread.start()
      for thread in threads:
        thread.join()

      # Dequeue every element using a single thread.
      results = []
      for _ in xrange(len(elems)):
        results.append(dequeued_t.eval())
      self.assertItemsEqual(elems, results) 

# 需要导入模块: import tensorflow [as 别名]
# 或者: from tensorflow import PaddingFIFOQueue [as 别名]
def testParallelDequeue(self):
    with self.test_session() as sess:
      q = tf.PaddingFIFOQueue(10, tf.float32, ((),))
      elems = [10.0, 20.0, 30.0, 40.0, 50.0, 60.0, 70.0, 80.0, 90.0, 100.0]
      enqueue_ops = [q.enqueue((x,)) for x in elems]
      dequeued_t = q.dequeue()

      # Enqueue every element using a single thread.
      for enqueue_op in enqueue_ops:
        enqueue_op.run()

      # Run one consumer thread for each element in elems.
      results = []

      def dequeue():
        results.append(sess.run(dequeued_t))
      threads = [self.checkedThread(target=dequeue) for _ in enqueue_ops]
      for thread in threads:
        thread.start()
      for thread in threads:
        thread.join()
      self.assertItemsEqual(elems, results) 

# 需要导入模块: import tensorflow [as 别名]
# 或者: from tensorflow import PaddingFIFOQueue [as 别名]
def testMultiEnqueueManyWithPartiallyKnownShapes(self):
    with self.test_session() as sess:
      q = tf.PaddingFIFOQueue(
          10, (tf.float32, tf.int32), shapes=((), (None,)))
      float_elems = [10.0, 20.0, 30.0, 40.0]
      int_elems = [[1, 2], [3, 4], [5, 6], [7, 8]]
      enqueue_op = q.enqueue_many((float_elems, int_elems))
      dequeued_t = q.dequeue()

      enqueue_op.run()
      enqueue_op.run()

      for i in range(8):
        float_val, int_val = sess.run(dequeued_t)
        self.assertEqual(float_elems[i % 4], float_val)
        self.assertAllEqual(int_elems[i % 4], int_val) 

# 需要导入模块: import tensorflow [as 别名]
# 或者: from tensorflow import PaddingFIFOQueue [as 别名]
def testEnqueueDequeueManyWrongPartiallyKnownShape(self):
    with self.test_session() as sess:
      # First dimension of second component is unknown, second
      # dimension must be 3.
      q = tf.PaddingFIFOQueue(10, (tf.int32, tf.int32), ((2, 2), (None, 3)))
      elems_ok = np.array([1] * 8).reshape((2, 2, 2)).astype(np.int32)
      elems_bad = tf.placeholder(tf.int32)
      enqueue_op = q.enqueue_many((elems_ok, elems_bad))
      dequeued_t = q.dequeue_many(2)
      with self.assertRaisesRegexp(
          tf.errors.InvalidArgumentError,
          "Shape mismatch in tuple component 1. "
          r"Expected \[2,\?,3\], got \[2,3,4\]"):
        sess.run([enqueue_op],
                 feed_dict={elems_bad: np.array([1] * 24).reshape((2, 3, 4))})
        dequeued_t.eval() 

# 需要导入模块: import tensorflow [as 别名]
# 或者: from tensorflow import PaddingFIFOQueue [as 别名]
def testParallelDequeueMany(self):
    with self.test_session() as sess:
      q = tf.PaddingFIFOQueue(1000, tf.float32, shapes=((),))
      elems = [10.0 * x for x in range(1000)]
      enqueue_op = q.enqueue_many((elems,))
      dequeued_t = q.dequeue_many(100)

      enqueue_op.run()

      # Dequeue 100 items in parallel on 10 threads.
      dequeued_elems = []

      def dequeue():
        dequeued_elems.extend(sess.run(dequeued_t))
      threads = [self.checkedThread(target=dequeue) for _ in range(10)]
      for thread in threads:
        thread.start()
      for thread in threads:
        thread.join()
      self.assertItemsEqual(elems, dequeued_elems) 

# 需要导入模块: import tensorflow [as 别名]
# 或者: from tensorflow import PaddingFIFOQueue [as 别名]
def testParallelDequeueUpTo(self):
    with self.test_session() as sess:
      q = tf.PaddingFIFOQueue(1000, tf.float32, shapes=((),))
      elems = [10.0 * x for x in range(1000)]
      enqueue_op = q.enqueue_many((elems,))
      close_op = q.close()
      dequeued_t = q.dequeue_up_to(101)

      enqueue_op.run()
      close_op.run()

      # Dequeue up to 101 items in parallel on 10 threads, from closed queue.
      dequeued_elems = []

      def dequeue():
        dequeued_elems.extend(sess.run(dequeued_t))
      threads = [self.checkedThread(target=dequeue) for _ in range(10)]
      for thread in threads:
        thread.start()
      for thread in threads:
        thread.join()
      self.assertItemsEqual(elems, dequeued_elems) 

# 需要导入模块: import tensorflow [as 别名]
# 或者: from tensorflow import PaddingFIFOQueue [as 别名]
def testDequeueManyWithTensorParameter(self):
    with self.test_session():
      # Define a first queue that contains integer counts.
      dequeue_counts = [random.randint(1, 10) for _ in range(100)]
      count_q = tf.PaddingFIFOQueue(100, tf.int32, ((),))
      enqueue_counts_op = count_q.enqueue_many((dequeue_counts,))
      total_count = sum(dequeue_counts)

      # Define a second queue that contains total_count elements.
      elems = [random.randint(0, 100) for _ in range(total_count)]
      q = tf.PaddingFIFOQueue(total_count, tf.int32, ((),))
      enqueue_elems_op = q.enqueue_many((elems,))

      # Define a subgraph that first dequeues a count, then DequeuesMany
      # that number of elements.
      dequeued_t = q.dequeue_many(count_q.dequeue())

      enqueue_counts_op.run()
      enqueue_elems_op.run()

      dequeued_elems = []
      for _ in dequeue_counts:
        dequeued_elems.extend(dequeued_t.eval())
      self.assertEqual(elems, dequeued_elems) 

# 需要导入模块: import tensorflow [as 别名]
# 或者: from tensorflow import PaddingFIFOQueue [as 别名]
def testDequeueFromClosedQueue(self):
    with self.test_session():
      q = tf.PaddingFIFOQueue(10, tf.float32, ((),))
      elems = [10.0, 20.0, 30.0, 40.0]
      enqueue_op = q.enqueue_many((elems,))
      close_op = q.close()
      dequeued_t = q.dequeue()

      enqueue_op.run()
      close_op.run()
      for elem in elems:
        self.assertEqual([elem], dequeued_t.eval())

      # Expect the operation to fail due to the queue being closed.
      with self.assertRaisesRegexp(tf.errors.OutOfRangeError,
                                   "is closed and has insufficient"):
        dequeued_t.eval() 

# 需要导入模块: import tensorflow [as 别名]
# 或者: from tensorflow import PaddingFIFOQueue [as 别名]
def testBlockingDequeueFromClosedQueue(self):
    with self.test_session() as sess:
      q = tf.PaddingFIFOQueue(10, tf.float32, ((),))
      elems = [10.0, 20.0, 30.0, 40.0]
      enqueue_op = q.enqueue_many((elems,))
      close_op = q.close()
      dequeued_t = q.dequeue()

      enqueue_op.run()

      def dequeue():
        for elem in elems:
          self.assertEqual([elem], sess.run(dequeued_t))
        # Expect the operation to fail due to the queue being closed.
        with self.assertRaisesRegexp(tf.errors.OutOfRangeError,
                                     "is closed and has insufficient"):
          sess.run(dequeued_t)

      dequeue_thread = self.checkedThread(target=dequeue)
      dequeue_thread.start()
      # The close_op should run after the dequeue_thread has blocked.
      # TODO(mrry): Figure out how to do this without sleeping.
      time.sleep(0.1)
      close_op.run()
      dequeue_thread.join() 

# 需要导入模块: import tensorflow [as 别名]
# 或者: from tensorflow import PaddingFIFOQueue [as 别名]
def testDequeueUpToFromClosedQueueReturnsRemainder(self):
    with self.test_session() as sess:
      q = tf.PaddingFIFOQueue(10, tf.float32, ((),))
      elems = [10.0, 20.0, 30.0, 40.0]
      enqueue_op = q.enqueue_many((elems,))
      close_op = q.close()
      dequeued_t = q.dequeue_up_to(3)

      enqueue_op.run()

      def dequeue():
        self.assertAllEqual(elems[:3], sess.run(dequeued_t))
        self.assertAllEqual(elems[3:], sess.run(dequeued_t))

      dequeue_thread = self.checkedThread(target=dequeue)
      dequeue_thread.start()
      # The close_op should run after the dequeue_thread has blocked.
      # TODO(mrry): Figure out how to do this without sleeping.
      time.sleep(0.1)
      close_op.run()
      dequeue_thread.join() 

# 需要导入模块: import tensorflow [as 别名]
# 或者: from tensorflow import PaddingFIFOQueue [as 别名]
def testBlockingDequeueManyFromClosedQueue(self):
    with self.test_session() as sess:
      q = tf.PaddingFIFOQueue(10, tf.float32, ((),))
      elems = [10.0, 20.0, 30.0, 40.0]
      enqueue_op = q.enqueue_many((elems,))
      close_op = q.close()
      dequeued_t = q.dequeue_many(4)

      enqueue_op.run()

      def dequeue():
        self.assertAllEqual(elems, sess.run(dequeued_t))
        # Expect the operation to fail due to the queue being closed.
        with self.assertRaisesRegexp(tf.errors.OutOfRangeError,
                                     "is closed and has insufficient"):
          sess.run(dequeued_t)

      dequeue_thread = self.checkedThread(target=dequeue)
      dequeue_thread.start()
      # The close_op should run after the dequeue_thread has blocked.
      # TODO(mrry): Figure out how to do this without sleeping.
      time.sleep(0.1)
      close_op.run()
      dequeue_thread.join() 

# 需要导入模块: import tensorflow [as 别名]
# 或者: from tensorflow import PaddingFIFOQueue [as 别名]
def testBlockingDequeueManyButNotAllFromClosedQueue(self):
    with self.test_session() as sess:
      q = tf.PaddingFIFOQueue(10, tf.float32, ((),))
      elems = [10.0, 20.0, 30.0, 40.0]
      enqueue_op = q.enqueue_many((elems,))
      close_op = q.close()
      dequeued_t = q.dequeue_many(3)

      enqueue_op.run()

      def dequeue():
        self.assertAllEqual(elems[:3], sess.run(dequeued_t))
        # Expect the operation to fail due to the queue being closed.
        with self.assertRaisesRegexp(tf.errors.OutOfRangeError,
                                     "is closed and has insufficient"):
          sess.run(dequeued_t)

      dequeue_thread = self.checkedThread(target=dequeue)
      dequeue_thread.start()
      # The close_op should run after the dequeue_thread has blocked.
      # TODO(mrry): Figure out how to do this without sleeping.
      time.sleep(0.1)
      close_op.run()
      dequeue_thread.join()