Python example_pb2.Example方法代码示例

# 需要导入模块: from tensorflow.core.example import example_pb2 [as 别名]
# 或者: from tensorflow.core.example.example_pb2 import Example [as 别名]
def _binary_to_text():
  reader = open(FLAGS.in_file, 'rb')
  writer = open(FLAGS.out_file, 'w')
  while True:
    len_bytes = reader.read(8)
    if not len_bytes:
      sys.stderr.write('Done reading\n')
      return
    str_len = struct.unpack('q', len_bytes)[0]
    tf_example_str = struct.unpack('%ds' % str_len, reader.read(str_len))[0]
    tf_example = example_pb2.Example.FromString(tf_example_str)
    examples = []
    for key in tf_example.features.feature:
      examples.append('%s=%s' % (key, tf_example.features.feature[key].bytes_list.value[0]))
    writer.write('%s\n' % '\t'.join(examples))
  reader.close()
  writer.close() 

# 需要导入模块: from tensorflow.core.example import example_pb2 [as 别名]
# 或者: from tensorflow.core.example.example_pb2 import Example [as 别名]
def _write_test_data():
        schema = feature_spec_to_schema({"f0": tf.VarLenFeature(dtype=tf.int64),
                                         "f1": tf.VarLenFeature(dtype=tf.int64),
                                         "f2": tf.VarLenFeature(dtype=tf.int64)})
        batches = [
            [1, 4, None],
            [2, None, None],
            [3, 5, None],
            [None, None, None],
        ]

        example_proto = [example_pb2.Example(features=feature_pb2.Features(feature={
            "f" + str(i): feature_pb2.Feature(int64_list=feature_pb2.Int64List(value=[f]))
            for i, f in enumerate(batch) if f is not None
        })) for batch in batches]

        return DataUtil.write_test_data(example_proto, schema) 

# 需要导入模块: from tensorflow.core.example import example_pb2 [as 别名]
# 或者: from tensorflow.core.example.example_pb2 import Example [as 别名]
def write_to_bin(self, news_dir, out_file):
        story_fnames = os.listdir(news_dir)
        num_stories = len(story_fnames)

        with open(out_file, 'wb') as writer:
            for idx,s in enumerate(story_fnames):
                #print(s)
                # Look in the tokenized story dirs to find the .story file corresponding to this url
                if os.path.isfile(os.path.join(news_dir, s)):
                    story_file = os.path.join(news_dir, s)
                article, abstract = self.get_art_abs(story_file)

                # Write to tf.Example
                if bytes(article, 'utf-8') == 0:
                    print('error!')
                tf_example = example_pb2.Example()
                tf_example.features.feature['article'].bytes_list.value.extend([bytes(article, 'utf-8') ])
                tf_example.features.feature['abstract'].bytes_list.value.extend([bytes(abstract, 'utf-8')])
                tf_example_str = tf_example.SerializeToString()
                str_len = len(tf_example_str)
                writer.write(struct.pack('q', str_len))
                writer.write(struct.pack('%ds' % str_len, tf_example_str))

        print("Finished writing file %s\n" % out_file)
        return story_fnames 

# 需要导入模块: from tensorflow.core.example import example_pb2 [as 别名]
# 或者: from tensorflow.core.example.example_pb2 import Example [as 别名]
def generate_image(image_shape, image_format='jpeg', label=0):
  """Generates an image and an example containing the encoded image.

  GenerateImage must be called within an active session.

  Args:
    image_shape: the shape of the image to generate.
    image_format: the encoding format of the image.
    label: the int64 labels for the image.

  Returns:
    image: the generated image.
    example: a TF-example with a feature key 'image/encoded' set to the
      serialized image and a feature key 'image/format' set to the image
      encoding format ['jpeg', 'png'].
  """
  image = np.random.random_integers(0, 255, size=image_shape)
  tf_encoded = _encoder(image, image_format)
  example = example_pb2.Example(features=feature_pb2.Features(feature={
      'image/encoded': _encoded_bytes_feature(tf_encoded),
      'image/format': _string_feature(image_format),
      'image/class/label': _encoded_int64_feature(np.array(label)),
  }))

  return image, example.SerializeToString() 

# 需要导入模块: from tensorflow.core.example import example_pb2 [as 别名]
# 或者: from tensorflow.core.example.example_pb2 import Example [as 别名]
def GenerateImage(self, image_format, image_shape):
    """Generates an image and an example containing the encoded image.

    Args:
      image_format: the encoding format of the image.
      image_shape: the shape of the image to generate.

    Returns:
      image: the generated image.
      example: a TF-example with a feature key 'image/encoded' set to the
        serialized image and a feature key 'image/format' set to the image
        encoding format ['jpeg', 'JPEG', 'png', 'PNG', 'raw'].
    """
    num_pixels = image_shape[0] * image_shape[1] * image_shape[2]
    image = np.linspace(
        0, num_pixels - 1, num=num_pixels).reshape(image_shape).astype(np.uint8)
    tf_encoded = self._Encoder(image, image_format)
    example = example_pb2.Example(features=feature_pb2.Features(feature={
        'image/encoded': self._EncodedBytesFeature(tf_encoded),
        'image/format': self._StringFeature(image_format)
    }))

    return image, example.SerializeToString() 

# 需要导入模块: from tensorflow.core.example import example_pb2 [as 别名]
# 或者: from tensorflow.core.example.example_pb2 import Example [as 别名]
def DecodeExample(self, serialized_example, item_handler, image_format):
    """Decodes the given serialized example with the specified item handler.

    Args:
      serialized_example: a serialized TF example string.
      item_handler: the item handler used to decode the image.
      image_format: the image format being decoded.

    Returns:
      the decoded image found in the serialized Example.
    """
    serialized_example = array_ops.reshape(serialized_example, shape=[])
    decoder = tfexample_decoder.TFExampleDecoder(
        keys_to_features={
            'image/encoded':
                parsing_ops.FixedLenFeature(
                    (), dtypes.string, default_value=''),
            'image/format':
                parsing_ops.FixedLenFeature(
                    (), dtypes.string, default_value=image_format),
        },
        items_to_handlers={'image': item_handler})
    [tf_image] = decoder.decode(serialized_example, ['image'])
    return tf_image 

# 需要导入模块: from tensorflow.core.example import example_pb2 [as 别名]
# 或者: from tensorflow.core.example.example_pb2 import Example [as 别名]
def testDecodeExampleWithFloatTensor(self):
    np_array = np.random.rand(2, 3, 1).astype('f')

    example = example_pb2.Example(features=feature_pb2.Features(feature={
        'array': self._EncodedFloatFeature(np_array),
    }))

    serialized_example = example.SerializeToString()

    with self.test_session():
      serialized_example = array_ops.reshape(serialized_example, shape=[])
      keys_to_features = {
          'array': parsing_ops.FixedLenFeature(np_array.shape, dtypes.float32)
      }
      items_to_handlers = {'array': tfexample_decoder.Tensor('array'),}
      decoder = tfexample_decoder.TFExampleDecoder(keys_to_features,
                                                   items_to_handlers)
      [tf_array] = decoder.decode(serialized_example, ['array'])
      self.assertAllEqual(tf_array.eval(), np_array) 

# 需要导入模块: from tensorflow.core.example import example_pb2 [as 别名]
# 或者: from tensorflow.core.example.example_pb2 import Example [as 别名]
def testDecodeExampleWithInt64Tensor(self):
    np_array = np.random.randint(1, 10, size=(2, 3, 1))

    example = example_pb2.Example(features=feature_pb2.Features(feature={
        'array': self._EncodedInt64Feature(np_array),
    }))

    serialized_example = example.SerializeToString()

    with self.test_session():
      serialized_example = array_ops.reshape(serialized_example, shape=[])
      keys_to_features = {
          'array': parsing_ops.FixedLenFeature(np_array.shape, dtypes.int64)
      }
      items_to_handlers = {'array': tfexample_decoder.Tensor('array'),}
      decoder = tfexample_decoder.TFExampleDecoder(keys_to_features,
                                                   items_to_handlers)
      [tf_array] = decoder.decode(serialized_example, ['array'])
      self.assertAllEqual(tf_array.eval(), np_array) 

# 需要导入模块: from tensorflow.core.example import example_pb2 [as 别名]
# 或者: from tensorflow.core.example.example_pb2 import Example [as 别名]
def testDecodeExampleWithFixLenTensorWithShape(self):
    np_array = np.array([[1, 2, 3], [4, 5, 6]])

    example = example_pb2.Example(features=feature_pb2.Features(feature={
        'labels': self._EncodedInt64Feature(np_array),
    }))

    serialized_example = example.SerializeToString()

    with self.test_session():
      serialized_example = array_ops.reshape(serialized_example, shape=[])
      keys_to_features = {
          'labels':
              parsing_ops.FixedLenFeature(
                  np_array.shape, dtype=dtypes.int64),
      }
      items_to_handlers = {
          'labels': tfexample_decoder.Tensor(
              'labels', shape=np_array.shape),
      }
      decoder = tfexample_decoder.TFExampleDecoder(keys_to_features,
                                                   items_to_handlers)
      [tf_labels] = decoder.decode(serialized_example, ['labels'])
      labels = tf_labels.eval()
      self.assertAllEqual(labels, np_array) 

# 需要导入模块: from tensorflow.core.example import example_pb2 [as 别名]
# 或者: from tensorflow.core.example.example_pb2 import Example [as 别名]
def testDecodeExampleWithVarLenTensorToDense(self):
    np_array = np.array([[1, 2, 3], [4, 5, 6]])
    example = example_pb2.Example(features=feature_pb2.Features(feature={
        'labels': self._EncodedInt64Feature(np_array),
    }))

    serialized_example = example.SerializeToString()

    with self.test_session():
      serialized_example = array_ops.reshape(serialized_example, shape=[])
      keys_to_features = {
          'labels': parsing_ops.VarLenFeature(dtype=dtypes.int64),
      }
      items_to_handlers = {
          'labels': tfexample_decoder.Tensor(
              'labels', shape=np_array.shape),
      }
      decoder = tfexample_decoder.TFExampleDecoder(keys_to_features,
                                                   items_to_handlers)
      [tf_labels] = decoder.decode(serialized_example, ['labels'])
      labels = tf_labels.eval()
      self.assertAllEqual(labels, np_array) 

# 需要导入模块: from tensorflow.core.example import example_pb2 [as 别名]
# 或者: from tensorflow.core.example.example_pb2 import Example [as 别名]
def testDecodeExampleWithSparseTensor(self):
    np_indices = np.array([[1], [2], [5]])
    np_values = np.array([0.1, 0.2, 0.6]).astype('f')
    example = example_pb2.Example(features=feature_pb2.Features(feature={
        'indices': self._EncodedInt64Feature(np_indices),
        'values': self._EncodedFloatFeature(np_values),
    }))

    serialized_example = example.SerializeToString()

    with self.test_session():
      serialized_example = array_ops.reshape(serialized_example, shape=[])
      keys_to_features = {
          'indices': parsing_ops.VarLenFeature(dtype=dtypes.int64),
          'values': parsing_ops.VarLenFeature(dtype=dtypes.float32),
      }
      items_to_handlers = {'labels': tfexample_decoder.SparseTensor(),}
      decoder = tfexample_decoder.TFExampleDecoder(keys_to_features,
                                                   items_to_handlers)
      [tf_labels] = decoder.decode(serialized_example, ['labels'])
      labels = tf_labels.eval()
      self.assertAllEqual(labels.indices, np_indices)
      self.assertAllEqual(labels.values, np_values)
      self.assertAllEqual(labels.dense_shape, np_values.shape) 

# 需要导入模块: from tensorflow.core.example import example_pb2 [as 别名]
# 或者: from tensorflow.core.example.example_pb2 import Example [as 别名]
def testDecodeJpegImage(self):
    image_tensor = np.random.randint(256, size=(4, 5, 3)).astype(np.uint8)
    encoded_jpeg = self._EncodeImage(image_tensor)
    decoded_jpeg = self._DecodeImage(encoded_jpeg)
    example = tf.train.Example(features=tf.train.Features(feature={
        'image/encoded': self._BytesFeature(encoded_jpeg),
        'image/format': self._BytesFeature('jpeg'),
        'image/source_id': self._BytesFeature('image_id'),
    })).SerializeToString()

    example_decoder = tf_example_decoder.TfExampleDecoder()
    tensor_dict = example_decoder.decode(tf.convert_to_tensor(example))

    self.assertAllEqual((tensor_dict[fields.InputDataFields.image].
                         get_shape().as_list()), [None, None, 3])
    with self.test_session() as sess:
      tensor_dict = sess.run(tensor_dict)

    self.assertAllEqual(decoded_jpeg, tensor_dict[fields.InputDataFields.image])
    self.assertEqual('image_id', tensor_dict[fields.InputDataFields.source_id]) 

# 需要导入模块: from tensorflow.core.example import example_pb2 [as 别名]
# 或者: from tensorflow.core.example.example_pb2 import Example [as 别名]
def testDecodeImageKeyAndFilename(self):
    image_tensor = np.random.randint(256, size=(4, 5, 3)).astype(np.uint8)
    encoded_jpeg = self._EncodeImage(image_tensor)
    example = tf.train.Example(features=tf.train.Features(feature={
        'image/encoded': self._BytesFeature(encoded_jpeg),
        'image/key/sha256': self._BytesFeature('abc'),
        'image/filename': self._BytesFeature('filename')
    })).SerializeToString()

    example_decoder = tf_example_decoder.TfExampleDecoder()
    tensor_dict = example_decoder.decode(tf.convert_to_tensor(example))

    with self.test_session() as sess:
      tensor_dict = sess.run(tensor_dict)

    self.assertEqual('abc', tensor_dict[fields.InputDataFields.key])
    self.assertEqual('filename', tensor_dict[fields.InputDataFields.filename]) 

# 需要导入模块: from tensorflow.core.example import example_pb2 [as 别名]
# 或者: from tensorflow.core.example.example_pb2 import Example [as 别名]
def testDecodePngImage(self):
    image_tensor = np.random.randint(256, size=(4, 5, 3)).astype(np.uint8)
    encoded_png = self._EncodeImage(image_tensor, encoding_type='png')
    decoded_png = self._DecodeImage(encoded_png, encoding_type='png')
    example = tf.train.Example(features=tf.train.Features(feature={
        'image/encoded': self._BytesFeature(encoded_png),
        'image/format': self._BytesFeature('png'),
        'image/source_id': self._BytesFeature('image_id')
    })).SerializeToString()

    example_decoder = tf_example_decoder.TfExampleDecoder()
    tensor_dict = example_decoder.decode(tf.convert_to_tensor(example))

    self.assertAllEqual((tensor_dict[fields.InputDataFields.image].
                         get_shape().as_list()), [None, None, 3])
    with self.test_session() as sess:
      tensor_dict = sess.run(tensor_dict)

    self.assertAllEqual(decoded_png, tensor_dict[fields.InputDataFields.image])
    self.assertEqual('image_id', tensor_dict[fields.InputDataFields.source_id]) 

# 需要导入模块: from tensorflow.core.example import example_pb2 [as 别名]
# 或者: from tensorflow.core.example.example_pb2 import Example [as 别名]
def testDecodeEmptyPngInstanceMasks(self):
    image_tensor = np.random.randint(256, size=(10, 10, 3)).astype(np.uint8)
    encoded_jpeg = self._EncodeImage(image_tensor)
    encoded_masks = []
    example = tf.train.Example(
        features=tf.train.Features(
            feature={
                'image/encoded': self._BytesFeature(encoded_jpeg),
                'image/format': self._BytesFeature('jpeg'),
                'image/object/mask': self._BytesFeature(encoded_masks),
                'image/height': self._Int64Feature([10]),
                'image/width': self._Int64Feature([10]),
            })).SerializeToString()

    example_decoder = tf_example_decoder.TfExampleDecoder(
        load_instance_masks=True, instance_mask_type=input_reader_pb2.PNG_MASKS)
    tensor_dict = example_decoder.decode(tf.convert_to_tensor(example))

    with self.test_session() as sess:
      tensor_dict = sess.run(tensor_dict)
      self.assertAllEqual(
          tensor_dict[fields.InputDataFields.groundtruth_instance_masks].shape,
          [0, 10, 10])