Python tensorflow.SequenceExample方法代码示例

# 需要导入模块: import tensorflow [as 别名]
# 或者: from tensorflow import SequenceExample [as 别名]
def _GetDecodeFunction(self, data_format: Union[Text, int],
                         schema: dataset_schema.Schema) -> Any:
    """Returns the decode function for `data_format`.

    Args:
      data_format: name of data format.
      schema: a dataset_schema.Schema for the data.

    Returns:
      Function for decoding examples.
    """
    if self._ShouldDecodeAsRawExample(data_format):
      if self._IsDataFormatSequenceExample(data_format):
        absl.logging.warning(
            'TFX Transform doesn\'t officially support tf.SequenceExample, '
            'follow b/38235367 to track official support progress. We do not '
            'guarantee not to break your pipeline if you use Transform with a '
            'tf.SequenceExample data type. Use at your own risk.')
      return lambda x: {RAW_EXAMPLE_KEY: x}
    else:
      return tft.coders.ExampleProtoCoder(schema, serialized=True).decode 

# 需要导入模块: import tensorflow [as 别名]
# 或者: from tensorflow import SequenceExample [as 别名]
def _PartitionFn(
    record: Union[tf.train.Example, tf.train.SequenceExample, bytes],
    num_partitions: int,
    buckets: List[int],
    split_config: example_gen_pb2.SplitConfig,
) -> int:
  """Partition function for the ExampleGen's output splits."""
  assert num_partitions == len(
      buckets), 'Partitions do not match bucket number.'
  partition_str = _GeneratePartitionKey(record, split_config)
  bucket = int(hashlib.sha256(partition_str).hexdigest(), 16) % buckets[-1]
  # For example, if buckets is [10,50,80], there will be 3 splits:
  #   bucket >=0 && < 10, returns 0
  #   bucket >=10 && < 50, returns 1
  #   bucket >=50 && < 80, returns 2
  return bisect.bisect(buckets, bucket) 

# 需要导入模块: import tensorflow [as 别名]
# 或者: from tensorflow import SequenceExample [as 别名]
def _WriteSplit(example_split: beam.pvalue.PCollection,
                output_split_path: Text) -> beam.pvalue.PDone:
  """Shuffles and writes output split as serialized records in TFRecord."""

  def _MaybeSerialize(x):
    if isinstance(x, (tf.train.Example, tf.train.SequenceExample)):
      return x.SerializeToString()
    return x

  return (example_split
          # TODO(jyzhao): make shuffle optional.
          | 'MaybeSerialize' >> beam.Map(_MaybeSerialize)
          | 'Shuffle' >> beam.transforms.Reshuffle()
          # TODO(jyzhao): multiple output format.
          | 'Write' >> beam.io.WriteToTFRecord(
              os.path.join(output_split_path, DEFAULT_FILE_NAME),
              file_name_suffix='.gz')) 

# 需要导入模块: import tensorflow [as 别名]
# 或者: from tensorflow import SequenceExample [as 别名]
def GetInputSourceToExamplePTransform(self) -> beam.PTransform:
    """Returns PTransform for converting input source to records.

    The record is by default assumed to be tf.train.Example protos, subclassses
    can serialize any protocol buffer into bytes as output PCollection,
    so long as the downstream component can consume it.

    Note that each input split will be transformed by this function separately.
    For complex use case, consider override 'GenerateExamplesByBeam' instead.

    Here is an example PTransform:
      @beam.ptransform_fn
      @beam.typehints.with_input_types(beam.Pipeline)
      @beam.typehints.with_output_types(Union[tf.train.Example,
                                              tf.train.SequenceExample,
                                              bytes])
      def ExamplePTransform(
          pipeline: beam.Pipeline,
          exec_properties: Dict[Text, Any],
          split_pattern: Text) -> beam.pvalue.PCollection
    """
    pass 

# 需要导入模块: import tensorflow [as 别名]
# 或者: from tensorflow import SequenceExample [as 别名]
def _images_to_example(image, image2):
  """Convert two consecutive images to SequenceExample."""
  example = tf.SequenceExample()
  feature_list = example.feature_lists.feature_list['moving_objs']
  feature = feature_list.feature.add()
  feature.float_list.value.extend(np.reshape(image, [-1]).tolist())
  feature = feature_list.feature.add()
  feature.float_list.value.extend(np.reshape(image2, [-1]).tolist())
  return example 

# 需要导入模块: import tensorflow [as 别名]
# 或者: from tensorflow import SequenceExample [as 别名]
def ReadInput(data_filepattern, shuffle, params):
  """Read the tf.SequenceExample tfrecord files.

  Args:
    data_filepattern: tf.SequenceExample tfrecord filepattern.
    shuffle: Whether to shuffle the examples.
    params: parameter dict.

  Returns:
    image sequence batch [batch_size, seq_len, image_size, image_size, channel].
  """
  image_size = params['image_size']
  filenames = tf.gfile.Glob(data_filepattern)
  filename_queue = tf.train.string_input_producer(filenames, shuffle=shuffle)
  reader = tf.TFRecordReader()
  _, example = reader.read(filename_queue)
  feature_sepc = {
      'moving_objs': tf.FixedLenSequenceFeature(
          shape=[image_size * image_size * 3], dtype=tf.float32)}
  _, features = tf.parse_single_sequence_example(
      example, sequence_features=feature_sepc)
  moving_objs = tf.reshape(
      features['moving_objs'], [params['seq_len'], image_size, image_size, 3])
  if shuffle:
    examples = tf.train.shuffle_batch(
        [moving_objs],
        batch_size=params['batch_size'],
        num_threads=64,
        capacity=params['batch_size'] * 100,
        min_after_dequeue=params['batch_size'] * 4)
  else:
    examples = tf.train.batch([moving_objs],
                              batch_size=params['batch_size'],
                              num_threads=16,
                              capacity=params['batch_size'])
  examples /= params['norm_scale']
  return examples 

# 需要导入模块: import tensorflow [as 别名]
# 或者: from tensorflow import SequenceExample [as 别名]
def _images_to_example(image, image2):
  """Convert 2 consecutive image to a SequenceExample."""
  example = tf.SequenceExample()
  feature_list = example.feature_lists.feature_list['moving_objs']
  feature = feature_list.feature.add()
  feature.float_list.value.extend(np.reshape(image, [-1]).tolist())
  feature = feature_list.feature.add()
  feature.float_list.value.extend(np.reshape(image2, [-1]).tolist())
  return example 

# 需要导入模块: import tensorflow [as 别名]
# 或者: from tensorflow import SequenceExample [as 别名]
def build_sequence_example_serving_input_receiver_fn(input_size,
                                                     context_feature_spec,
                                                     example_feature_spec,
                                                     default_batch_size=None):
  """Creates a serving_input_receiver_fn for `SequenceExample` inputs.

  A string placeholder is used for inputs. Note that the context_feature_spec
  and example_feature_spec shouldn't contain weights, labels or training
  only features in general.

  Args:
    input_size: (int) The number of frames to keep in a SequenceExample. If
      specified, truncation or padding may happen. Otherwise, set it to None to
      allow dynamic list size (recommended).
    context_feature_spec: (dict) Map from feature keys to `FixedLenFeature` or
      `VarLenFeature` values.
    example_feature_spec: (dict) Map from  feature keys to `FixedLenFeature` or
      `VarLenFeature` values.
    default_batch_size: (int) Number of query examples expected per batch. Leave
      unset for variable batch size (recommended).

  Returns:
    A `tf.estimator.export.ServingInputReceiver` object, which packages the
    placeholders and the resulting feature Tensors together.
  """
  return build_ranking_serving_input_receiver_fn(
      SEQ,
      context_feature_spec,
      example_feature_spec,
      list_size=input_size,
      receiver_name="sequence_example",
      default_batch_size=default_batch_size) 

# 需要导入模块: import tensorflow [as 别名]
# 或者: from tensorflow import SequenceExample [as 别名]
def _GeneratePartitionKey(record: Union[tf.train.Example,
                                        tf.train.SequenceExample, bytes],
                          split_config: example_gen_pb2.SplitConfig) -> bytes:
  """Generates key for partition."""

  if not split_config.HasField('partition_feature_name'):
    if isinstance(record, bytes):
      return record
    return record.SerializeToString(deterministic=True)

  if isinstance(record, tf.train.Example):
    features = record.features.feature  # pytype: disable=attribute-error
  elif isinstance(record, tf.train.SequenceExample):
    features = record.context.feature  # pytype: disable=attribute-error
  else:
    raise RuntimeError('Split by `partition_feature_name` is only supported '
                       'for FORMAT_TF_EXAMPLE and FORMAT_TF_SEQUENCE_EXAMPLE '
                       'payload format.')

  # Use a feature for partitioning the examples.
  feature_name = split_config.partition_feature_name
  if feature_name not in features:
    raise RuntimeError('Feature name `{}` does not exist.'.format(feature_name))
  feature = features[feature_name]
  if not feature.HasField('kind'):
    raise RuntimeError('Partition feature does not contain any value.')
  if (not feature.HasField('bytes_list') and
      not feature.HasField('int64_list')):
    raise RuntimeError('Only `bytes_list` and `int64_list` features are '
                       'supported for partition.')
  return feature.SerializeToString(deterministic=True) 

# 需要导入模块: import tensorflow [as 别名]
# 或者: from tensorflow import SequenceExample [as 别名]
def bytestring_to_record(example):
    """Convert a serialized tf.SequenceExample to Python-friendly objects.

    Parameters
    ----------
    example : str
        A single serialized tf.SequenceExample

    Returns
    -------
    features : np.array, shape=(n, 128)
        Array of feature coefficients over time (axis=0).

    meta : pd.DataFrame, len=n
        Corresponding labels and metadata for these features.
    """
    rec = tf.train.SequenceExample.FromString(example)
    start_time = rec.context.feature[START_TIME].float_list.value[0]
    vid_id = rec.context.feature[VIDEO_ID].bytes_list.value[0].decode('utf-8')
    labels = list(rec.context.feature[LABELS].int64_list.value)
    data = rec.feature_lists.feature_list[AUDIO_EMBEDDING_FEATURE_NAME]
    features = [b.bytes_list.value for b in data.feature]
    features = np.asarray([np.frombuffer(_[0], dtype=np.uint8)
                           for _ in features])
    if features.ndim == 1:
        raise ValueError("Caught unexpected feature shape: {}"
                         .format(features.shape))

    rows = [{VIDEO_ID: vid_id, LABELS: labels, TIME: np.uint16(start_time + t)}
            for t in range(len(features))]

    return features, pd.DataFrame.from_records(data=rows) 

# 需要导入模块: import tensorflow [as 别名]
# 或者: from tensorflow import SequenceExample [as 别名]
def __init__(self, keys=None, prefix=None, return_dense=True,
               default_value=-1.0):
    """Initialize the bounding box handler.

    Args:
      keys: A list of four key names representing the ymin, xmin, ymax, xmax
        in the Example or SequenceExample.
      prefix: An optional prefix for each of the bounding box keys in the
        Example or SequenceExample. If provided, `prefix` is prepended to each
        key in `keys`.
      return_dense: if True, returns a dense tensor; if False, returns as
        sparse tensor.
      default_value: The value used when the `tensor_key` is not found in a
        particular `TFExample`.

    Raises:
      ValueError: if keys is not `None` and also not a list of exactly 4 keys
    """
    if keys is None:
      keys = ['ymin', 'xmin', 'ymax', 'xmax']
    elif len(keys) != 4:
      raise ValueError('BoundingBoxSequence expects 4 keys but got {}'.format(
          len(keys)))
    self._prefix = prefix
    self._keys = keys
    self._full_keys = [prefix + k for k in keys]
    self._return_dense = return_dense
    self._default_value = default_value
    super(BoundingBoxSequence, self).__init__(self._full_keys) 

# 需要导入模块: import tensorflow [as 别名]
# 或者: from tensorflow import SequenceExample [as 别名]
def decode(self, tf_seq_example_string_tensor, items=None):
    """Decodes serialized tf.SequenceExample and returns a tensor dictionary.

    Args:
      tf_seq_example_string_tensor: A string tensor holding a serialized
        tensorflow example proto.
      items: The list of items to decode. These must be a subset of the item
        keys in self._items_to_handlers. If `items` is left as None, then all
        of the items in self._items_to_handlers are decoded.

    Returns:
      A dictionary of the following tensors.
      fields.InputDataFields.image - 3D uint8 tensor of shape [None, None, seq]
        containing image(s).
      fields.InputDataFields.source_id - string tensor containing original
        image id.
      fields.InputDataFields.key - string tensor with unique sha256 hash key.
      fields.InputDataFields.filename - string tensor with original dataset
        filename.
      fields.InputDataFields.groundtruth_boxes - 2D float32 tensor of shape
        [None, 4] containing box corners.
      fields.InputDataFields.groundtruth_classes - 1D int64 tensor of shape
        [None] containing classes for the boxes.
      fields.InputDataFields.groundtruth_area - 1D float32 tensor of shape
        [None] containing object mask area in pixel squared.
      fields.InputDataFields.groundtruth_is_crowd - 1D bool tensor of shape
        [None] indicating if the boxes enclose a crowd.
      fields.InputDataFields.groundtruth_difficult - 1D bool tensor of shape
        [None] indicating if the boxes represent `difficult` instances.
    """
    serialized_example = tf.reshape(tf_seq_example_string_tensor, shape=[])
    decoder = TFSequenceExampleDecoderHelper(self.keys_to_context_features,
                                             self.keys_to_features,
                                             self.items_to_handlers)
    if not items:
      items = decoder.list_items()
    tensors = decoder.decode(serialized_example, items=items)
    tensor_dict = dict(zip(items, tensors))

    return tensor_dict