Python tensorflow.parse_single_sequence_example方法代码示例

# 需要导入模块: import tensorflow [as 别名]
# 或者: from tensorflow import parse_single_sequence_example [as 别名]
def _read_single_sequence_example(file_list, tokens_shape=None):
  """Reads and parses SequenceExamples from TFRecord-encoded file_list."""
  tf.logging.info('Constructing TFRecordReader from files: %s', file_list)
  file_queue = tf.train.string_input_producer(file_list)
  reader = tf.TFRecordReader()
  seq_key, serialized_record = reader.read(file_queue)
  ctx, sequence = tf.parse_single_sequence_example(
      serialized_record,
      sequence_features={
          data_utils.SequenceWrapper.F_TOKEN_ID:
              tf.FixedLenSequenceFeature(tokens_shape or [], dtype=tf.int64),
          data_utils.SequenceWrapper.F_LABEL:
              tf.FixedLenSequenceFeature([], dtype=tf.int64),
          data_utils.SequenceWrapper.F_WEIGHT:
              tf.FixedLenSequenceFeature([], dtype=tf.float32),
      })
  return seq_key, ctx, sequence 

# 需要导入模块: import tensorflow [as 别名]
# 或者: from tensorflow import parse_single_sequence_example [as 别名]
def _parse_record(example_proto):
        context_features = {
            "length": tf.FixedLenFeature([], dtype=tf.int64)
        }
        sequence_features = {
            "tokens": tf.FixedLenSequenceFeature([], dtype=tf.int64),
            "labels": tf.FixedLenSequenceFeature([], dtype=tf.int64)
        }

        context_parsed, sequence_parsed = tf.parse_single_sequence_example(serialized=example_proto,
            context_features=context_features, sequence_features=sequence_features)

        return context_parsed['length'], sequence_parsed['tokens'], sequence_parsed['labels']

    # Read training data from TFRecord file, shuffle, loop over data infinitely and
    # pad to the longest sentence 

# 需要导入模块: import tensorflow [as 别名]
# 或者: from tensorflow import parse_single_sequence_example [as 别名]
def parse_sequence_example(serialized_example, num_views):
  """Parses a serialized sequence example into views, sequence length data."""
  context_features = {
      'task': tf.FixedLenFeature(shape=[], dtype=tf.string),
      'len': tf.FixedLenFeature(shape=[], dtype=tf.int64)
  }
  view_names = ['view%d' % i for i in range(num_views)]
  fixed_features = [
      tf.FixedLenSequenceFeature(
          shape=[], dtype=tf.string) for _ in range(len(view_names))]
  sequence_features = dict(zip(view_names, fixed_features))
  context_parse, sequence_parse = tf.parse_single_sequence_example(
      serialized=serialized_example,
      context_features=context_features,
      sequence_features=sequence_features)
  views = tf.stack([sequence_parse[v] for v in view_names])
  lens = [sequence_parse[v].get_shape().as_list()[0] for v in view_names]
  assert len(set(lens)) == 1
  seq_len = tf.shape(sequence_parse[v])[0]
  return context_parse, views, seq_len 

# 需要导入模块: import tensorflow [as 别名]
# 或者: from tensorflow import parse_single_sequence_example [as 别名]
def _assign_queue(self, proto_text):
        """
        Args:
            proto_text: object to be enqueued and managed by parallel threads.
        """

        with tf.variable_scope('shuffle_queue'):
            queue = tf.RandomShuffleQueue(
                capacity=self.capacity,
                min_after_dequeue=10*self.batch_size,
                dtypes=tf.string, shapes=[()])

            enqueue_op = queue.enqueue(proto_text)
            example_dq = queue.dequeue()

            qr = tf.train.QueueRunner(queue, [enqueue_op] * 4)
            tf.train.add_queue_runner(qr)

            _sequence_lengths, _sequences = tf.parse_single_sequence_example(
                serialized=example_dq,
                context_features=LENGTHS,
                sequence_features=SEQUENCES)
        return _sequence_lengths, _sequences 

# 需要导入模块: import tensorflow [as 别名]
# 或者: from tensorflow import parse_single_sequence_example [as 别名]
def deserialize_fasta_sequence(example):
    context = {
        'protein_length': tf.FixedLenFeature([1], tf.int64),
        'id': tf.FixedLenFeature([], tf.string)
    }

    features = {
        'primary': tf.FixedLenSequenceFeature([1], tf.int64),
    }

    context, features = tf.parse_single_sequence_example(
        example,
        context_features=context,
        sequence_features=features
    )

    return {'id': context['id'],
            'primary': tf.to_int32(features['primary'][:, 0]),
            'protein_length': tf.to_int32(context['protein_length'][0])} 

# 需要导入模块: import tensorflow [as 别名]
# 或者: from tensorflow import parse_single_sequence_example [as 别名]
def parse_sequence_example(serialized_example, num_views):
  """Parses a serialized sequence example into views, sequence length data."""
  context_features = {
      'task': tf.FixedLenFeature(shape=[], dtype=tf.string),
      'len': tf.FixedLenFeature(shape=[], dtype=tf.int64)
  }
  view_names = ['view%d' % i for i in range(num_views)]
  fixed_features = [
      tf.FixedLenSequenceFeature(
          shape=[], dtype=tf.string) for _ in range(len(view_names))]
  sequence_features = dict(zip(view_names, fixed_features))
  context_parse, sequence_parse = tf.parse_single_sequence_example(
      serialized=serialized_example,
      context_features=context_features,
      sequence_features=sequence_features)
  views = tf.stack([sequence_parse[v] for v in view_names])
  lens = [sequence_parse[v].get_shape().as_list()[0] for v in view_names]
  assert len(set(lens)) == 1
  seq_len = tf.shape(sequence_parse[view_names[-1]])[0]
  return context_parse, views, seq_len 

# 需要导入模块: import tensorflow [as 别名]
# 或者: from tensorflow import parse_single_sequence_example [as 别名]
def _read_sequence_example(filename_queue,
                           n_labels=50, n_samples=59049, n_segments=10):
  reader = tf.TFRecordReader()
  _, serialized_example = reader.read(filename_queue)
  context, sequence = tf.parse_single_sequence_example(
    serialized_example,
    context_features={
      'raw_labels': tf.FixedLenFeature([], dtype=tf.string)
    },
    sequence_features={
      'raw_segments': tf.FixedLenSequenceFeature([], dtype=tf.string)
    })

  segments = tf.decode_raw(sequence['raw_segments'], tf.float32)
  segments.set_shape([n_segments, n_samples])

  labels = tf.decode_raw(context['raw_labels'], tf.uint8)
  labels.set_shape([n_labels])
  labels = tf.cast(labels, tf.float32)

  return segments, labels 

# 需要导入模块: import tensorflow [as 别名]
# 或者: from tensorflow import parse_single_sequence_example [as 别名]
def parse_fn(serialized_example):
  """Parse a serialized example."""
  
  # user_id is not currently used.
  context_features = {
    'user_id': tf.FixedLenFeature([], dtype=tf.int64)
  }
  sequence_features = {
    'movie_ids': tf.FixedLenSequenceFeature([], dtype=tf.int64)
  }
  parsed_feature, parsed_sequence_feature = tf.parse_single_sequence_example(
    serialized=serialized_example,
    context_features=context_features,
    sequence_features=sequence_features
  )
  movie_ids = parsed_sequence_feature['movie_ids']
  return movie_ids 

# 需要导入模块: import tensorflow [as 别名]
# 或者: from tensorflow import parse_single_sequence_example [as 别名]
def read_dataset(filename, num_channels=39):
    """Read data from tfrecord file."""

    def parse_fn(example_proto):
        """Parse function for reading single sequence example."""
        sequence_features = {
            'inputs': tf.FixedLenSequenceFeature(shape=[num_channels], dtype=tf.float32),
            'labels': tf.FixedLenSequenceFeature(shape=[], dtype=tf.string)
        }

        context, sequence = tf.parse_single_sequence_example(
            serialized=example_proto,
            sequence_features=sequence_features
        )

        return sequence['inputs'], sequence['labels']

    dataset = tf.data.TFRecordDataset(filename)
    dataset = dataset.map(parse_fn)

    return dataset 

# 需要导入模块: import tensorflow [as 别名]
# 或者: from tensorflow import parse_single_sequence_example [as 别名]
def example_parser(self, filename_queue):
        reader = tf.TFRecordReader()
        key, record_string = reader.read(filename_queue)
        features = {
            'labels': tf.FixedLenSequenceFeature([], tf.int64),
            'tokens': tf.FixedLenSequenceFeature([], tf.int64),
            'shapes': tf.FixedLenSequenceFeature([], tf.int64),
            'chars': tf.FixedLenSequenceFeature([], tf.int64),
            'seq_len': tf.FixedLenSequenceFeature([], tf.int64),
            'tok_len': tf.FixedLenSequenceFeature([], tf.int64),
        }

        _, example = tf.parse_single_sequence_example(serialized=record_string, sequence_features=features)
        labels = example['labels']
        tokens = example['tokens']
        shapes = example['shapes']
        chars = example['chars']
        seq_len = example['seq_len']
        tok_len = example['tok_len']
        # context = c['context']
        return labels, tokens, shapes, chars, seq_len, tok_len
        # return labels, tokens, labels, labels, labels 

# 需要导入模块: import tensorflow [as 别名]
# 或者: from tensorflow import parse_single_sequence_example [as 别名]
def ner_example_parser(filename_queue):
    reader = tf.TFRecordReader()
    key, record_string = reader.read(filename_queue)

    # Define how to parse the example
    context_features = {
        'seq_len': tf.FixedLenFeature([], tf.int64),
    }
    sequence_features = {
        "tokens": tf.FixedLenSequenceFeature([], dtype=tf.int64),
        "ner_labels": tf.FixedLenSequenceFeature([], dtype=tf.int64),
        "entities": tf.FixedLenSequenceFeature([], dtype=tf.int64),
    }
    context_parsed, sequence_parsed = tf.parse_single_sequence_example(serialized=record_string,
                                                                       context_features=context_features,
                                                                       sequence_features=sequence_features)
    tokens = sequence_parsed['tokens']
    ner_labels = sequence_parsed['ner_labels']
    entities = sequence_parsed['entities']
    seq_len = context_parsed['seq_len']

    return [tokens, ner_labels, entities, seq_len] 

# 需要导入模块: import tensorflow [as 别名]
# 或者: from tensorflow import parse_single_sequence_example [as 别名]
def parse_sentence(serialized):
  """Parses a tensorflow.SequenceExample into an caption.

  Args:
    serialized: A scalar string Tensor; a single serialized SequenceExample.

  Returns:
    key: The keywords in a sentence.
    num_key: The number of keywords.
    sentence: A description.
    sentence_length: The length of the description.
  """
  context, sequence = tf.parse_single_sequence_example(
    serialized,
    context_features={},
    sequence_features={
      'key': tf.FixedLenSequenceFeature([], dtype=tf.int64),
      'sentence': tf.FixedLenSequenceFeature([], dtype=tf.int64),
    })
  key = tf.to_int32(sequence['key'])
  key = tf.random_shuffle(key)
  sentence = tf.to_int32(sequence['sentence'])
  return key, tf.shape(key)[0], sentence, tf.shape(sentence)[0] 

# 需要导入模块: import tensorflow [as 别名]
# 或者: from tensorflow import parse_single_sequence_example [as 别名]
def parse_image(serialized, tf):
  """Parses a tensorflow.SequenceExample into an image and detected objects.

  Args:
    serialized: A scalar string Tensor; a single serialized SequenceExample.

  Returns:
    encoded_image: A scalar string Tensor containing a JPEG encoded image.
    classes: A 1-D int64 Tensor containing the detected objects.
    scores: A 1-D float32 Tensor containing the detection scores.
  """
  context, sequence = tf.parse_single_sequence_example(
    serialized,
    sequence_features={
      'classes': tf.FixedLenSequenceFeature([], dtype=tf.int64),
      'scores': tf.FixedLenSequenceFeature([], dtype=tf.float32),
    })

  classes = tf.to_int32(sequence['classes'])
  scores = sequence['scores']
  return classes, scores 

# 需要导入模块: import tensorflow [as 别名]
# 或者: from tensorflow import parse_single_sequence_example [as 别名]
def parse_image(serialized):
  """Parses a tensorflow.SequenceExample into an image and detected objects.

  Args:
    serialized: A scalar string Tensor; a single serialized SequenceExample.

  Returns:
    name: A scalar string Tensor containing the image name.
    classes: A 1-D int64 Tensor containing the detected objects.
    scores: A 1-D float32 Tensor containing the detection scores.
  """
  context, sequence = tf.parse_single_sequence_example(
    serialized,
    context_features={
      'image/name': tf.FixedLenFeature([], dtype=tf.string)
    },
    sequence_features={
      'classes': tf.FixedLenSequenceFeature([], dtype=tf.int64),
      'scores': tf.FixedLenSequenceFeature([], dtype=tf.float32),
    })

  name = context['image/name']
  classes = tf.to_int32(sequence['classes'])
  scores = sequence['scores']
  return name, classes, scores 

# 需要导入模块: import tensorflow [as 别名]
# 或者: from tensorflow import parse_single_sequence_example [as 别名]
def parse_image(serialized):
  """Parses a tensorflow.SequenceExample into an image and detected objects.

  Args:
    serialized: A scalar string Tensor; a single serialized SequenceExample.

  Returns:
    encoded_image: A scalar string Tensor containing a JPEG encoded image.
    classes: A 1-D int64 Tensor containing the detected objects.
    scores: A 1-D float32 Tensor containing the detection scores.
  """
  context, sequence = tf.parse_single_sequence_example(
    serialized,
    context_features={
      'image/data': tf.FixedLenFeature([], dtype=tf.string)
    },
    sequence_features={
      'classes': tf.FixedLenSequenceFeature([], dtype=tf.int64),
      'scores': tf.FixedLenSequenceFeature([], dtype=tf.float32),
    })

  encoded_image = context['image/data']
  classes = tf.to_int32(sequence['classes'])
  scores = sequence['scores']
  return encoded_image, classes, scores