Python tensorflow.TextLineReader方法代码示例

# 需要导入模块: import tensorflow [as 别名]
# 或者: from tensorflow import TextLineReader [as 别名]
def read_instances(self, count, shuffle, epochs):
    """Reads the data represented by this DataSource using a TensorFlow reader.

    Arguments:
      epochs: The number of epochs or passes over the data to perform.
    Returns:
      A tensor containing instances that are read.
    """
    # None implies unlimited; switch the value to None when epochs is 0.
    epochs = epochs or None

    files = tf.train.match_filenames_once(self._path, name='files')
    queue = tf.train.string_input_producer(files, num_epochs=epochs, shuffle=shuffle,
                                           name='queue')
    reader = tf.TextLineReader(name='reader')
    _, instances = reader.read_up_to(queue, count, name='read')

    return instances 

# 需要导入模块: import tensorflow [as 别名]
# 或者: from tensorflow import TextLineReader [as 别名]
def _voc_seg_load_file(path, epochs=None, shuffle=True, seed=0):

    PASCAL_ROOT = os.environ['VOC_DIR']
    filename_queue = tf.train.string_input_producer([path],
            num_epochs=epochs, shuffle=shuffle, seed=seed)

    reader = tf.TextLineReader()
    key, value = reader.read(filename_queue)
    image_path, seg_path = tf.decode_csv(value, record_defaults=[[''], ['']], field_delim=' ')

    image_abspath = PASCAL_ROOT + image_path
    seg_abspath = PASCAL_ROOT + seg_path

    image_content = tf.read_file(image_abspath)
    image = decode_image(image_content, channels=3)
    image.set_shape([None, None, 3])

    imgshape = tf.shape(image)[:2]
    imgname = image_path

    seg_content = tf.read_file(seg_abspath)
    seg = tf.cast(tf.image.decode_png(seg_content, channels=1), tf.int32)
    return image, seg, imgshape, imgname 

# 需要导入模块: import tensorflow [as 别名]
# 或者: from tensorflow import TextLineReader [as 别名]
def _imagenet_load_file(path, epochs=None, shuffle=True, seed=0, subset='train', prepare_path=True):
    IMAGENET_ROOT = os.environ.get('IMAGENET_DIR', '')
    if not isinstance(path, list):
        path = [path]
    filename_queue = tf.train.string_input_producer(path,
            num_epochs=epochs, shuffle=shuffle, seed=seed)

    reader = tf.TextLineReader()
    key, value = reader.read(filename_queue)
    image_path, label_str = tf.decode_csv(value, record_defaults=[[''], ['']], field_delim=' ')

    if prepare_path:
        image_abspath = IMAGENET_ROOT + '/images/' + subset + image_path
    else:
        image_abspath = image_path

    image_content = tf.read_file(image_abspath)
    image = decode_image(image_content, channels=3)
    image.set_shape([None, None, 3])

    imgshape = tf.shape(image)[:2]
    label = tf.string_to_number(label_str, out_type=tf.int32)

    return image, label, imgshape, image_path 

# 需要导入模块: import tensorflow [as 别名]
# 或者: from tensorflow import TextLineReader [as 别名]
def _relpath_no_label_load_file(path, root_path, epochs=None, shuffle=True, seed=0):
    filename_queue = tf.train.string_input_producer([path],
            num_epochs=epochs, shuffle=shuffle, seed=seed)

    reader = tf.TextLineReader()
    key, value = reader.read(filename_queue)
    #image_path, = tf.decode_csv(value, record_defaults=[['']], field_delim=' ')
    image_path = value

    image_abspath = root_path + '/' + image_path

    image_content = tf.read_file(image_abspath)
    image = decode_image(image_content, channels=3)
    image.set_shape([None, None, 3])

    imgshape = tf.shape(image)[:2]

    return image, imgshape, image_path 

# 需要导入模块: import tensorflow [as 别名]
# 或者: from tensorflow import TextLineReader [as 别名]
def _abspath_no_label_load_file(path, epochs=None, shuffle=True, seed=0):
    filename_queue = tf.train.string_input_producer([path],
            num_epochs=epochs, shuffle=shuffle, seed=seed)

    reader = tf.TextLineReader()
    key, value = reader.read(filename_queue)
    #image_path, = tf.decode_csv(value, record_defaults=[['']], field_delim=' ')
    image_path = value

    image_abspath = image_path

    image_content = tf.read_file(image_abspath)
    image = decode_image(image_content, channels=3)
    image.set_shape([None, None, 3])

    imgshape = tf.shape(image)[:2]

    return image, imgshape, image_path 

# 需要导入模块: import tensorflow [as 别名]
# 或者: from tensorflow import TextLineReader [as 别名]
def _testOneEpoch(self, files):
    with self.test_session() as sess:
      reader = tf.TextLineReader(name="test_reader")
      queue = tf.FIFOQueue(99, [tf.string], shapes=())
      key, value = reader.read(queue)

      queue.enqueue_many([files]).run()
      queue.close().run()
      for i in range(self._num_files):
        for j in range(self._num_lines):
          k, v = sess.run([key, value])
          self.assertAllEqual("%s:%d" % (files[i], j + 1), tf.compat.as_text(k))
          self.assertAllEqual(self._LineText(i, j), v)

      with self.assertRaisesOpError("is closed and has insufficient elements "
                                    "\\(requested 1, current size 0\\)"):
        k, v = sess.run([key, value]) 

# 需要导入模块: import tensorflow [as 别名]
# 或者: from tensorflow import TextLineReader [as 别名]
def testSkipHeaderLines(self):
    files = self._CreateFiles()
    with self.test_session() as sess:
      reader = tf.TextLineReader(skip_header_lines=1, name="test_reader")
      queue = tf.FIFOQueue(99, [tf.string], shapes=())
      key, value = reader.read(queue)

      queue.enqueue_many([files]).run()
      queue.close().run()
      for i in range(self._num_files):
        for j in range(self._num_lines - 1):
          k, v = sess.run([key, value])
          self.assertAllEqual("%s:%d" % (files[i], j + 2), tf.compat.as_text(k))
          self.assertAllEqual(self._LineText(i, j + 1), v)

      with self.assertRaisesOpError("is closed and has insufficient elements "
                                    "\\(requested 1, current size 0\\)"):
        k, v = sess.run([key, value]) 

# 需要导入模块: import tensorflow [as 别名]
# 或者: from tensorflow import TextLineReader [as 别名]
def testManagedEndOfInputOneQueue(self):
    # Tests that the supervisor finishes without an error when using
    # a fixed number of epochs, reading from a single queue.
    logdir = _test_dir("managed_end_of_input_one_queue")
    os.makedirs(logdir)
    data_path = self._csv_data(logdir)
    with tf.Graph().as_default():
      # Create an input pipeline that reads the file 3 times.
      filename_queue = tf.train.string_input_producer([data_path], num_epochs=3)
      reader = tf.TextLineReader()
      _, csv = reader.read(filename_queue)
      rec = tf.decode_csv(csv, record_defaults=[[1], [1], [1]])
      sv = tf.train.Supervisor(logdir=logdir)
      with sv.managed_session("") as sess:
        while not sv.should_stop():
          sess.run(rec) 

# 需要导入模块: import tensorflow [as 别名]
# 或者: from tensorflow import TextLineReader [as 别名]
def testManagedEndOfInputTwoQueues(self):
    # Tests that the supervisor finishes without an error when using
    # a fixed number of epochs, reading from two queues, the second
    # one producing a batch from the first one.
    logdir = _test_dir("managed_end_of_input_two_queues")
    os.makedirs(logdir)
    data_path = self._csv_data(logdir)
    with tf.Graph().as_default():
      # Create an input pipeline that reads the file 3 times.
      filename_queue = tf.train.string_input_producer([data_path], num_epochs=3)
      reader = tf.TextLineReader()
      _, csv = reader.read(filename_queue)
      rec = tf.decode_csv(csv, record_defaults=[[1], [1], [1]])
      shuff_rec = tf.train.shuffle_batch(rec, 1, 6, 4)
      sv = tf.train.Supervisor(logdir=logdir)
      with sv.managed_session("") as sess:
        while not sv.should_stop():
          sess.run(shuff_rec) 

# 需要导入模块: import tensorflow [as 别名]
# 或者: from tensorflow import TextLineReader [as 别名]
def _read_image_and_box(self, bboxes_csv):
        """Extract the filename from the queue, read the image and
        produce a single box
        Returns:
            image, [y_min, x_min, y_max, x_max, label]
        """

        reader = tf.TextLineReader(skip_header_lines=True)
        _, row = reader.read(bboxes_csv)
        # file ,y_min, x_min, y_max, x_max, label
        record_defaults = [[""], [0.], [0.], [0.], [0.], [0.]]
        # eg:
        # 2008_000033,0.1831831831831832,0.208,0.7717717717717718,0.952,0
        filename, y_min, x_min, y_max, x_max, label = tf.decode_csv(
            row, record_defaults)
        image_path = os.path.join(self._data_dir, 'VOCdevkit', 'VOC2012',
                                  'JPEGImages') + "/" + filename + ".jpg"

        # image is normalized in [-1,1]
        image = read_image_jpg(image_path)
        return image, tf.stack([y_min, x_min, y_max, x_max, label]) 

# 需要导入模块: import tensorflow [as 别名]
# 或者: from tensorflow import TextLineReader [as 别名]
def _read_image_and_box(self, bboxes_csv):
        """Extract the filename from the queue, read the image and
        produce a single box
        Returns:
            image, box
        """

        reader = tf.TextLineReader(skip_header_lines=True)
        _, row = reader.read(bboxes_csv)
        # file ,y_min, x_min, y_max, x_max, label
        record_defaults = [[""], [0.], [0.], [0.], [0.], [0.]]
        # eg:
        # 2008_000033,0.1831831831831832,0.208,0.7717717717717718,0.952,0
        filename, y_min, x_min, y_max, x_max, label = tf.decode_csv(
            row, record_defaults)
        image_path = os.path.join(self._data_dir, 'VOCdevkit', 'VOC2012',
                                  'JPEGImages') + "/" + filename + ".jpg"

        # image is normalized in [-1,1], convert to #_image_depth depth
        image = read_image_jpg(image_path, depth=self._image_depth)
        return image, tf.stack([y_min, x_min, y_max, x_max, label]) 

# 需要导入模块: import tensorflow [as 别名]
# 或者: from tensorflow import TextLineReader [as 别名]
def load_files(filename_queue):
    """
    Read and parse examples from data files.

    Args:
        filename: A list of string: filenames to read from

    Returns:
        uint8image: a [height, width, depth] uint8 Tensor with the image data
        label: a int32 Tensor
    """

    line_reader = tf.TextLineReader()
    key, line = line_reader.read(filename_queue)
    label, image_path = tf.decode_csv(records=line,
                                      record_defaults=[tf.constant([], dtype=tf.int32), tf.constant([], dtype=tf.string)],
                                      field_delim=' ')
    file_contents = tf.read_file(image_path)
    image = tf.image.decode_jpeg(file_contents, channels=3)

    return image, label 

# 需要导入模块: import tensorflow [as 别名]
# 或者: from tensorflow import TextLineReader [as 别名]
def read_csv(filename, directory, num_epoch=None, records=None):
    if records is None:
        records = records_default
    # Read examples and labels from file
    filename_queue = tf.train.string_input_producer([os.path.join(directory, filename)],
                                                    num_epochs=num_epoch, shuffle=True)
    reader = tf.TextLineReader()
    _, value = reader.read(filename_queue)
    decoded = tf.decode_csv(value, record_defaults=records)
    im_path = tf.stack(decoded[0])
    if len(decoded) > 1:
        label = tf.stack(decoded[1:])
    else:
        label = [0]
    return im_path, label 

# 需要导入模块: import tensorflow [as 别名]
# 或者: from tensorflow import TextLineReader [as 别名]
def __init__(self, config, batch_size, one_hot=False):
        self.lookup = None
        reader = tf.TextLineReader()
        filename_queue = tf.train.string_input_producer(["chargan.txt"])
        key, x = reader.read(filename_queue)
        vocabulary = self.get_vocabulary()

        table = tf.contrib.lookup.string_to_index_table_from_tensor(
            mapping = vocabulary, default_value = 0)

        x = tf.string_join([x, tf.constant(" " * 64)]) 
        x = tf.substr(x, [0], [64])
        x = tf.string_split(x,delimiter='')
        x = tf.sparse_tensor_to_dense(x, default_value=' ')
        x = tf.reshape(x, [64])
        x = table.lookup(x)
        self.one_hot = one_hot
        if one_hot:
            x = tf.one_hot(x, len(vocabulary))
            x = tf.cast(x, dtype=tf.float32)
            x = tf.reshape(x, [1, int(x.get_shape()[0]), int(x.get_shape()[1]), 1])
        else:
            x = tf.cast(x, dtype=tf.float32)
            x -= len(vocabulary)/2.0
            x /= len(vocabulary)/2.0
            x = tf.reshape(x, [1,1, 64, 1])

        num_preprocess_threads = 8

        x = tf.train.shuffle_batch(
          [x],
          batch_size=batch_size,
          num_threads=num_preprocess_threads,
          capacity= 5000,
          min_after_dequeue=500,
          enqueue_many=True)

        self.x = x
        self.table = table 

# 需要导入模块: import tensorflow [as 别名]
# 或者: from tensorflow import TextLineReader [as 别名]
def input_fn(name="input", tables="", num_epochs=None, num_workers=1, worker_id=0, capacity=0, batch_size=64):
    with tf.variable_scope(name_or_scope=name, reuse=False) as scope:
        with tf.device(device_name_or_function = ("/job:localhost/replica:0/task:%d"%worker_id) if worker_id != -1 else None):
            filename_queue = tf.train.string_input_producer(tables, num_epochs=num_epochs)
            reader = tf.TextLineReader()
            keys, values = reader.read_up_to(filename_queue, batch_size)
            batch_keys, batch_values = tf.train.batch(
                [keys, values],
                batch_size=batch_size,
                capacity=10 * batch_size,
                enqueue_many=True,
                num_threads=1)
            record_defaults = [['']] * 4 + [[-1]] + [['']] * 9
            data = tf.decode_csv(batch_values, record_defaults=record_defaults, field_delim=';')

            pageid = data[4]
            ctr = data[7]
            cvr = data[8]
            price = data[9]
            isclick = data[10]
            pay = data[11]

            ctr = _parse_dense_features(ctr, (-1, 50))
            cvr = _parse_dense_features(cvr, (-1, 50))
            price = _parse_dense_features(price, (-1, 50))
            isclick = _parse_dense_features(isclick, (-1, 50))
            pay = _parse_dense_features(pay, (-1, 50))

            batch_data = {'keys': batch_keys,
                        'pageid': pageid,
                        'ctr': ctr,
                        'cvr': cvr,
                        'price': price,
                        'click': isclick,
                        'pay': pay}
    return batch_data