Python tensorflow.FixedLenSequenceFeature方法代码示例

# 需要导入模块: import tensorflow [as 别名]
# 或者: from tensorflow import FixedLenSequenceFeature [as 别名]
def _mapper(example_proto):
  features = {
      'samples': tf.FixedLenSequenceFeature([1], tf.float32, allow_missing=True),
      'label': tf.FixedLenSequenceFeature([], tf.string, allow_missing=True)
  }
  example = tf.parse_single_example(example_proto, features)

  wav = example['samples'][:, 0]

  wav = wav[:16384]
  wav_len = tf.shape(wav)[0]
  wav = tf.pad(wav, [[0, 16384 - wav_len]])

  label = tf.reduce_join(example['label'], 0)

  return wav, label 

# 需要导入模块: import tensorflow [as 别名]
# 或者: from tensorflow import FixedLenSequenceFeature [as 别名]
def _read_single_sequence_example(file_list, tokens_shape=None):
  """Reads and parses SequenceExamples from TFRecord-encoded file_list."""
  tf.logging.info('Constructing TFRecordReader from files: %s', file_list)
  file_queue = tf.train.string_input_producer(file_list)
  reader = tf.TFRecordReader()
  seq_key, serialized_record = reader.read(file_queue)
  ctx, sequence = tf.parse_single_sequence_example(
      serialized_record,
      sequence_features={
          data_utils.SequenceWrapper.F_TOKEN_ID:
              tf.FixedLenSequenceFeature(tokens_shape or [], dtype=tf.int64),
          data_utils.SequenceWrapper.F_LABEL:
              tf.FixedLenSequenceFeature([], dtype=tf.int64),
          data_utils.SequenceWrapper.F_WEIGHT:
              tf.FixedLenSequenceFeature([], dtype=tf.float32),
      })
  return seq_key, ctx, sequence 

# 需要导入模块: import tensorflow [as 别名]
# 或者: from tensorflow import FixedLenSequenceFeature [as 别名]
def example_reading_spec(self):
    data_fields, data_items_to_decoders = (
        super(ImageVqav2Tokens10kLabels3k, self).example_reading_spec())
    data_fields["image/image_id"] = tf.FixedLenFeature((), tf.int64)
    data_fields["image/question_id"] = tf.FixedLenFeature((), tf.int64)
    data_fields["image/question"] = tf.FixedLenSequenceFeature(
        (), tf.int64, allow_missing=True)
    data_fields["image/answer"] = tf.FixedLenSequenceFeature(
        (), tf.int64, allow_missing=True)

    data_items_to_decoders[
        "question"] = tf.contrib.slim.tfexample_decoder.Tensor(
            "image/question")
    data_items_to_decoders[
        "targets"] = tf.contrib.slim.tfexample_decoder.Tensor(
            "image/answer")
    return data_fields, data_items_to_decoders 

# 需要导入模块: import tensorflow [as 别名]
# 或者: from tensorflow import FixedLenSequenceFeature [as 别名]
def _parse_record(example_proto):
        context_features = {
            "length": tf.FixedLenFeature([], dtype=tf.int64)
        }
        sequence_features = {
            "tokens": tf.FixedLenSequenceFeature([], dtype=tf.int64),
            "labels": tf.FixedLenSequenceFeature([], dtype=tf.int64)
        }

        context_parsed, sequence_parsed = tf.parse_single_sequence_example(serialized=example_proto,
            context_features=context_features, sequence_features=sequence_features)

        return context_parsed['length'], sequence_parsed['tokens'], sequence_parsed['labels']

    # Read training data from TFRecord file, shuffle, loop over data infinitely and
    # pad to the longest sentence 

# 需要导入模块: import tensorflow [as 别名]
# 或者: from tensorflow import FixedLenSequenceFeature [as 别名]
def parse_sequence_example(serialized_example, num_views):
  """Parses a serialized sequence example into views, sequence length data."""
  context_features = {
      'task': tf.FixedLenFeature(shape=[], dtype=tf.string),
      'len': tf.FixedLenFeature(shape=[], dtype=tf.int64)
  }
  view_names = ['view%d' % i for i in range(num_views)]
  fixed_features = [
      tf.FixedLenSequenceFeature(
          shape=[], dtype=tf.string) for _ in range(len(view_names))]
  sequence_features = dict(zip(view_names, fixed_features))
  context_parse, sequence_parse = tf.parse_single_sequence_example(
      serialized=serialized_example,
      context_features=context_features,
      sequence_features=sequence_features)
  views = tf.stack([sequence_parse[v] for v in view_names])
  lens = [sequence_parse[v].get_shape().as_list()[0] for v in view_names]
  assert len(set(lens)) == 1
  seq_len = tf.shape(sequence_parse[v])[0]
  return context_parse, views, seq_len 

# 需要导入模块: import tensorflow [as 别名]
# 或者: from tensorflow import FixedLenSequenceFeature [as 别名]
def deserialize_fasta_sequence(example):
    context = {
        'protein_length': tf.FixedLenFeature([1], tf.int64),
        'id': tf.FixedLenFeature([], tf.string)
    }

    features = {
        'primary': tf.FixedLenSequenceFeature([1], tf.int64),
    }

    context, features = tf.parse_single_sequence_example(
        example,
        context_features=context,
        sequence_features=features
    )

    return {'id': context['id'],
            'primary': tf.to_int32(features['primary'][:, 0]),
            'protein_length': tf.to_int32(context['protein_length'][0])} 

# 需要导入模块: import tensorflow [as 别名]
# 或者: from tensorflow import FixedLenSequenceFeature [as 别名]
def parse_sequence_example(serialized_example, num_views):
  """Parses a serialized sequence example into views, sequence length data."""
  context_features = {
      'task': tf.FixedLenFeature(shape=[], dtype=tf.string),
      'len': tf.FixedLenFeature(shape=[], dtype=tf.int64)
  }
  view_names = ['view%d' % i for i in range(num_views)]
  fixed_features = [
      tf.FixedLenSequenceFeature(
          shape=[], dtype=tf.string) for _ in range(len(view_names))]
  sequence_features = dict(zip(view_names, fixed_features))
  context_parse, sequence_parse = tf.parse_single_sequence_example(
      serialized=serialized_example,
      context_features=context_features,
      sequence_features=sequence_features)
  views = tf.stack([sequence_parse[v] for v in view_names])
  lens = [sequence_parse[v].get_shape().as_list()[0] for v in view_names]
  assert len(set(lens)) == 1
  seq_len = tf.shape(sequence_parse[view_names[-1]])[0]
  return context_parse, views, seq_len 

# 需要导入模块: import tensorflow [as 别名]
# 或者: from tensorflow import FixedLenSequenceFeature [as 别名]
def testSequenceExampleListWithInconsistentDataFails(self):
    original = sequence_example(feature_lists=feature_lists({
        "a": feature_list([
            int64_feature([-1, 0]), float_feature([2, 3])
        ])
    }))

    serialized = original.SerializeToString()

    self._test(
        {
            "example_name": "in1",
            "serialized": tf.convert_to_tensor(serialized),
            "sequence_features": {"a": tf.FixedLenSequenceFeature(
                (2,), tf.int64)}
        },
        expected_err=(tf.OpError, "Feature list: a, Index: 1."
                      "  Data types don't match. Expected type: int64")) 

# 需要导入模块: import tensorflow [as 别名]
# 或者: from tensorflow import FixedLenSequenceFeature [as 别名]
def testSequenceExampleListWithWrongDataTypeFails(self):
    original = sequence_example(feature_lists=feature_lists({
        "a": feature_list([
            float_feature([2, 3])
        ])
    }))

    serialized = original.SerializeToString()

    self._test(
        {
            "example_name": "in1",
            "serialized": tf.convert_to_tensor(serialized),
            "sequence_features": {"a": tf.FixedLenSequenceFeature(
                (2,), tf.int64)}
        },
        expected_err=(tf.OpError,
                      "Feature list: a, Index: 0.  Data types don't match."
                      " Expected type: int64")) 

# 需要导入模块: import tensorflow [as 别名]
# 或者: from tensorflow import FixedLenSequenceFeature [as 别名]
def testSequenceExampleListWithWrongShapeFails(self):
    original = sequence_example(feature_lists=feature_lists({
        "a": feature_list([
            int64_feature([2, 3]), int64_feature([2, 3, 4])
        ]),
    }))

    serialized = original.SerializeToString()

    self._test(
        {
            "example_name": "in1",
            "serialized": tf.convert_to_tensor(serialized),
            "sequence_features": {"a": tf.FixedLenSequenceFeature(
                (2,), tf.int64)}
        },
        expected_err=(tf.OpError, r"Name: in1, Key: a, Index: 1."
                      r"  Number of int64 values != expected."
                      r"  values size: 3 but output shape: \[2\]")) 

# 需要导入模块: import tensorflow [as 别名]
# 或者: from tensorflow import FixedLenSequenceFeature [as 别名]
def testSequenceExampleWithMissingFeatureListFails(self):
    original = sequence_example(feature_lists=feature_lists({}))

    # Test fails because we didn't add:
    #  feature_list_dense_defaults = {"a": None}
    self._test(
        {
            "example_name": "in1",
            "serialized": tf.convert_to_tensor(original.SerializeToString()),
            "sequence_features": {"a": tf.FixedLenSequenceFeature(
                (2,), tf.int64)}
        },
        expected_err=(
            tf.OpError,
            "Name: in1, Feature list 'a' is required but could not be found."
            "  Did you mean to include it in"
            " feature_list_dense_missing_assumed_empty or"
            " feature_list_dense_defaults?")) 

# 需要导入模块: import tensorflow [as 别名]
# 或者: from tensorflow import FixedLenSequenceFeature [as 别名]
def create_tensorrec_dataset_from_tfrecord(tfrecord_path):
    """
    Loads a TFRecord file and creates a Dataset with the contents.
    :param tfrecord_path: str
    :return: tf.data.Dataset
    """

    def parse_tensorrec_tfrecord(example_proto):
        features = {
            'row_index': tf.FixedLenSequenceFeature((), tf.int64, allow_missing=True),
            'col_index': tf.FixedLenSequenceFeature((), tf.int64, allow_missing=True),
            'values': tf.FixedLenSequenceFeature((), tf.float32, allow_missing=True),
            'd0': tf.FixedLenFeature((), tf.int64),
            'd1': tf.FixedLenFeature((), tf.int64),
        }
        parsed_features = tf.parse_single_example(example_proto, features)
        return (parsed_features['row_index'], parsed_features['col_index'], parsed_features['values'],
                parsed_features['d0'], parsed_features['d1'])

    dataset = tf.data.TFRecordDataset(tfrecord_path).map(parse_tensorrec_tfrecord)
    return dataset 

# 需要导入模块: import tensorflow [as 别名]
# 或者: from tensorflow import FixedLenSequenceFeature [as 别名]
def _read_sequence_example(filename_queue,
                           n_labels=50, n_samples=59049, n_segments=10):
  reader = tf.TFRecordReader()
  _, serialized_example = reader.read(filename_queue)
  context, sequence = tf.parse_single_sequence_example(
    serialized_example,
    context_features={
      'raw_labels': tf.FixedLenFeature([], dtype=tf.string)
    },
    sequence_features={
      'raw_segments': tf.FixedLenSequenceFeature([], dtype=tf.string)
    })

  segments = tf.decode_raw(sequence['raw_segments'], tf.float32)
  segments.set_shape([n_segments, n_samples])

  labels = tf.decode_raw(context['raw_labels'], tf.uint8)
  labels.set_shape([n_labels])
  labels = tf.cast(labels, tf.float32)

  return segments, labels 

# 需要导入模块: import tensorflow [as 别名]
# 或者: from tensorflow import FixedLenSequenceFeature [as 别名]
def parse_fn(serialized_example):
  """Parse a serialized example."""
  
  # user_id is not currently used.
  context_features = {
    'user_id': tf.FixedLenFeature([], dtype=tf.int64)
  }
  sequence_features = {
    'movie_ids': tf.FixedLenSequenceFeature([], dtype=tf.int64)
  }
  parsed_feature, parsed_sequence_feature = tf.parse_single_sequence_example(
    serialized=serialized_example,
    context_features=context_features,
    sequence_features=sequence_features
  )
  movie_ids = parsed_sequence_feature['movie_ids']
  return movie_ids 

# 需要导入模块: import tensorflow [as 别名]
# 或者: from tensorflow import FixedLenSequenceFeature [as 别名]
def read_dataset(filename, num_channels=39):
    """Read data from tfrecord file."""

    def parse_fn(example_proto):
        """Parse function for reading single sequence example."""
        sequence_features = {
            'inputs': tf.FixedLenSequenceFeature(shape=[num_channels], dtype=tf.float32),
            'labels': tf.FixedLenSequenceFeature(shape=[], dtype=tf.string)
        }

        context, sequence = tf.parse_single_sequence_example(
            serialized=example_proto,
            sequence_features=sequence_features
        )

        return sequence['inputs'], sequence['labels']

    dataset = tf.data.TFRecordDataset(filename)
    dataset = dataset.map(parse_fn)

    return dataset