Python lookup_ops.index_table_from_tensor函数代码示例

  def test_index_table_from_tensor_with_invalid_hashers(self):
    with self.test_session():
      with self.assertRaises(TypeError):
        lookup_ops.index_table_from_tensor(
            vocabulary_list=["brain", "salad", "surgery"],
            num_oov_buckets=1,
            hasher_spec=1)

      table = lookup_ops.index_table_from_tensor(
          vocabulary_list=["brain", "salad", "surgery"],
          num_oov_buckets=1,
          hasher_spec=lookup_ops.HasherSpec("my-awesome-hash", None))

      self.assertRaises(ValueError, table.lookup,
                        constant_op.constant(["salad", "surgery", "tarkus"]))

 def _process_labels(self, labels):
   if labels is None:
     raise ValueError(
         'You must provide a labels Tensor. Given: None. '
         'Suggested troubleshooting steps: Check that your data contain '
         'your label feature. Check that your input_fn properly parses and '
         'returns labels.')
   if isinstance(labels, sparse_tensor.SparseTensor):
     if labels.dtype == dtypes.string:
       label_ids_values = lookup_ops.index_table_from_tensor(
           vocabulary_list=tuple(self._label_vocabulary),
           name='class_id_lookup').lookup(labels.values)
       label_ids = sparse_tensor.SparseTensor(
           indices=labels.indices,
           values=label_ids_values,
           dense_shape=labels.dense_shape)
     else:
       label_ids = labels
     return math_ops.to_int64(
         sparse_ops.sparse_to_indicator(label_ids, self._n_classes))
   msg = ('labels shape must be [batch_size, {}]. '
          'Given: ').format(self._n_classes)
   labels_shape = array_ops.shape(labels)
   check_rank_op = control_flow_ops.Assert(
       math_ops.equal(array_ops.rank(labels), 2),
       data=[msg, labels_shape])
   check_label_dim = control_flow_ops.Assert(
       math_ops.equal(labels_shape[-1], self._n_classes),
       data=[msg, labels_shape])
   with ops.control_dependencies([check_rank_op, check_label_dim]):
     return array_ops.identity(labels)

 def create_loss(self, features, mode, logits, labels):
   """See `Head`."""
   del mode  # Unused for this head.
   logits = ops.convert_to_tensor(logits)
   labels = _check_dense_labels_match_logits_and_reshape(
       labels=labels, logits=logits, expected_labels_dimension=1)
   if self._label_vocabulary is not None:
     labels = lookup_ops.index_table_from_tensor(
         vocabulary_list=tuple(self._label_vocabulary),
         name='class_id_lookup').lookup(labels)
   labels = math_ops.to_float(labels)
   labels = _assert_range(labels, 2)
   unweighted_loss = nn.sigmoid_cross_entropy_with_logits(
       labels=labels, logits=logits)
   weights = _get_weights_and_check_match_logits(
       features=features, weight_column=self._weight_column, logits=logits)
   weighted_sum_loss = losses.compute_weighted_loss(
       unweighted_loss, weights=weights, reduction=losses.Reduction.SUM)
   # _weights() can return 1.
   example_weight_sum = math_ops.reduce_sum(
       weights * array_ops.ones_like(unweighted_loss))
   return LossSpec(
       weighted_sum_loss=weighted_sum_loss,
       example_weight_sum=example_weight_sum,
       processed_labels=labels)

 def _process_labels(self, labels):
   if isinstance(labels, sparse_tensor.SparseTensor):
     if labels.dtype == dtypes.string:
       label_ids_values = lookup_ops.index_table_from_tensor(
           vocabulary_list=tuple(self._label_vocabulary),
           name='class_id_lookup').lookup(labels.values)
       label_ids = sparse_tensor.SparseTensor(
           indices=labels.indices,
           values=label_ids_values,
           dense_shape=labels.dense_shape)
     else:
       label_ids = labels
     return math_ops.to_int64(
         sparse_ops.sparse_to_indicator(label_ids, self._n_classes))
   msg = ('labels shape must be [batch_size, {}]. '
          'Given: ').format(self._n_classes)
   labels_shape = array_ops.shape(labels)
   check_rank_op = control_flow_ops.Assert(
       math_ops.equal(array_ops.rank(labels), 2),
       data=[msg, labels_shape])
   check_label_dim = control_flow_ops.Assert(
       math_ops.equal(labels_shape[-1], self._n_classes),
       data=[msg, labels_shape])
   with ops.control_dependencies([check_rank_op, check_label_dim]):
     return array_ops.identity(labels)

  def testDecodeExampleWithBranchedLookup(self):

    example = example_pb2.Example(features=feature_pb2.Features(feature={
        'image/object/class/text': self._BytesFeatureFromList(
            np.array(['cat', 'dog', 'guinea pig'])),
    }))
    serialized_example = example.SerializeToString()
    # 'dog' -> 0, 'guinea pig' -> 1, 'cat' -> 2
    table = lookup_ops.index_table_from_tensor(
        constant_op.constant(['dog', 'guinea pig', 'cat']))

    with self.test_session() as sess:
      sess.run(lookup_ops.tables_initializer())

      serialized_example = array_ops.reshape(serialized_example, shape=[])

      keys_to_features = {
          'image/object/class/text': parsing_ops.VarLenFeature(dtypes.string),
      }

      items_to_handlers = {
          'labels':
              tf_example_decoder.LookupTensor('image/object/class/text', table),
      }

      decoder = slim_example_decoder.TFExampleDecoder(keys_to_features,
                                                      items_to_handlers)
      obtained_class_ids = decoder.decode(serialized_example)[0].eval()

    self.assertAllClose([2, 0, 1], obtained_class_ids)

 def test_index_table_from_tensor_empty_vocabulary_list(self):
   with self.test_session():
     table = lookup_ops.index_table_from_tensor(
         vocabulary_list=np.array([], dtype=np.str_), num_oov_buckets=1)
     ids = table.lookup(constant_op.constant(["salad", "surgery", "brain"]))
     self.assertRaises(errors_impl.OpError, ids.eval)
     with self.assertRaisesRegexp(
         errors_impl.OpError, "keys and values cannot be empty"):
       lookup_ops.tables_initializer().run()

  def test_index_table_from_tensor_with_tensor_init(self):
    with self.test_session():
      table = lookup_ops.index_table_from_tensor(
          vocabulary_list=("brain", "salad", "surgery"), num_oov_buckets=1)
      ids = table.lookup(constant_op.constant(("salad", "surgery", "tarkus")))

      self.assertRaises(errors_impl.OpError, ids.eval)
      lookup_ops.tables_initializer().run()
      self.assertAllEqual((1, 2, 3), ids.eval())

  def test_int64_index_table_from_tensor_with_tensor_init(self):
    with self.test_session():
      table = lookup_ops.index_table_from_tensor(
          vocabulary_list=(42, 1, -1000), num_oov_buckets=1, dtype=dtypes.int64)
      ids = table.lookup(
          constant_op.constant((1, -1000, 11), dtype=dtypes.int64))

      self.assertRaises(errors_impl.OpError, ids.eval)
      lookup_ops.tables_initializer().run()
      self.assertAllEqual((1, 2, 3), ids.eval())

  def test_index_table_from_tensor_with_default_value(self):
    default_value = -42
    with self.test_session():
      table = lookup_ops.index_table_from_tensor(
          vocabulary_list=["brain", "salad", "surgery"],
          default_value=default_value)
      ids = table.lookup(constant_op.constant(["salad", "surgery", "tarkus"]))

      self.assertRaises(errors_impl.OpError, ids.eval)
      lookup_ops.tables_initializer().run()
      self.assertAllEqual((1, 2, default_value), ids.eval())

def create_test_iterator(hparams, mode):
  """Create test iterator."""
  src_vocab_table = lookup_ops.index_table_from_tensor(
      tf.constant([hparams.eos, "a", "b", "c", "d"]))
  tgt_vocab_mapping = tf.constant([hparams.sos, hparams.eos, "a", "b", "c"])
  tgt_vocab_table = lookup_ops.index_table_from_tensor(tgt_vocab_mapping)
  if mode == tf.contrib.learn.ModeKeys.INFER:
    reverse_tgt_vocab_table = lookup_ops.index_to_string_table_from_tensor(
        tgt_vocab_mapping)

  src_dataset = tf.contrib.data.Dataset.from_tensor_slices(
      tf.constant(["a a b b c", "a b b"]))

  if mode != tf.contrib.learn.ModeKeys.INFER:
    tgt_dataset = tf.contrib.data.Dataset.from_tensor_slices(
        tf.constant(["a b c b c", "a b c b"]))
    return (
        iterator_utils.get_iterator(
            src_dataset=src_dataset,
            tgt_dataset=tgt_dataset,
            src_vocab_table=src_vocab_table,
            tgt_vocab_table=tgt_vocab_table,
            batch_size=hparams.batch_size,
            sos=hparams.sos,
            eos=hparams.eos,
            source_reverse=hparams.source_reverse,
            random_seed=hparams.random_seed,
            num_buckets=hparams.num_buckets),
        src_vocab_table,
        tgt_vocab_table)
  else:
    return (
        iterator_utils.get_infer_iterator(
            src_dataset=src_dataset,
            src_vocab_table=src_vocab_table,
            eos=hparams.eos,
            source_reverse=hparams.source_reverse,
            batch_size=hparams.batch_size),
        src_vocab_table,
        tgt_vocab_table,
        reverse_tgt_vocab_table)

  def testDecodeExampleWithBranchedBackupHandler(self):
    example1 = example_pb2.Example(
        features=feature_pb2.Features(
            feature={
                'image/object/class/text':
                    self._BytesFeatureFromList(
                        np.array(['cat', 'dog', 'guinea pig'])),
                'image/object/class/label':
                    self._Int64FeatureFromList(np.array([42, 10, 900]))
            }))
    example2 = example_pb2.Example(
        features=feature_pb2.Features(
            feature={
                'image/object/class/text':
                    self._BytesFeatureFromList(
                        np.array(['cat', 'dog', 'guinea pig'])),
            }))
    example3 = example_pb2.Example(
        features=feature_pb2.Features(
            feature={
                'image/object/class/label':
                    self._Int64FeatureFromList(np.array([42, 10, 901]))
            }))
    # 'dog' -> 0, 'guinea pig' -> 1, 'cat' -> 2
    table = lookup_ops.index_table_from_tensor(
        constant_op.constant(['dog', 'guinea pig', 'cat']))
    keys_to_features = {
        'image/object/class/text': parsing_ops.VarLenFeature(dtypes.string),
        'image/object/class/label': parsing_ops.VarLenFeature(dtypes.int64),
    }
    backup_handler = tf_example_decoder.BackupHandler(
        handler=slim_example_decoder.Tensor('image/object/class/label'),
        backup=tf_example_decoder.LookupTensor('image/object/class/text',
                                               table))
    items_to_handlers = {
        'labels': backup_handler,
    }
    decoder = slim_example_decoder.TFExampleDecoder(keys_to_features,
                                                    items_to_handlers)
    obtained_class_ids_each_example = []
    with self.test_session() as sess:
      sess.run(lookup_ops.tables_initializer())
      for example in [example1, example2, example3]:
        serialized_example = array_ops.reshape(
            example.SerializeToString(), shape=[])
        obtained_class_ids_each_example.append(
            decoder.decode(serialized_example)[0].eval())

    self.assertAllClose([42, 10, 900], obtained_class_ids_each_example[0])
    self.assertAllClose([2, 0, 1], obtained_class_ids_each_example[1])
    self.assertAllClose([42, 10, 901], obtained_class_ids_each_example[2])

 def create_loss(self, features, mode, logits, labels):
   """See `Head`."""
   del mode, features  # Unused for this head.
   labels = _check_and_reshape_dense_labels(labels, self.logits_dimension)
   if self._label_vocabulary is not None:
     labels = lookup_ops.index_table_from_tensor(
         vocabulary_list=tuple(self._label_vocabulary),
         name='class_id_lookup').lookup(labels)
   labels = math_ops.to_float(labels)
   labels = _assert_range(labels, 2)
   return LossAndLabels(
       unweighted_loss=nn.sigmoid_cross_entropy_with_logits(
           labels=labels, logits=logits),
       processed_labels=labels)

 def _label_ids(self, labels):
   """Converts labels to integer id space."""
   if self._label_vocabulary is None:
     if not labels.dtype.is_integer:
       raise ValueError('Labels dtype should be integer '
                        'Instead got %s.' % labels.dtype)
     label_ids = labels
   else:
     if labels.dtype != dtypes.string:
       raise ValueError('Labels dtype should be string if there is a '
                        'vocabulary. Instead got {}'.format(labels.dtype))
     label_ids = lookup_ops.index_table_from_tensor(
         vocabulary_list=tuple(self._label_vocabulary),
         name='class_id_lookup').lookup(labels)
   return _assert_range(label_ids, self._n_classes)

  def testGetInferIterator(self):
    src_vocab_table = lookup_ops.index_table_from_tensor(
        tf.constant(["a", "b", "c", "eos", "sos"]))
    src_dataset = tf.data.Dataset.from_tensor_slices(
        tf.constant(["c c a", "c a", "d", "f e a g"]))
    hparams = tf.contrib.training.HParams(
        random_seed=3,
        eos="eos",
        sos="sos")
    batch_size = 2
    src_max_len = 3
    iterator = iterator_utils.get_infer_iterator(
        src_dataset=src_dataset,
        src_vocab_table=src_vocab_table,
        batch_size=batch_size,
        eos=hparams.eos,
        src_max_len=src_max_len)
    table_initializer = tf.tables_initializer()
    source = iterator.source
    seq_len = iterator.source_sequence_length
    self.assertEqual([None, None], source.shape.as_list())
    self.assertEqual([None], seq_len.shape.as_list())
    with self.test_session() as sess:
      sess.run(table_initializer)
      sess.run(iterator.initializer)

      (source_v, seq_len_v) = sess.run((source, seq_len))
      self.assertAllEqual(
          [[2, 2, 0],   # c c a
           [2, 0, 3]],  # c a eos
          source_v)
      self.assertAllEqual([3, 2], seq_len_v)

      (source_v, seq_len_v) = sess.run((source, seq_len))
      self.assertAllEqual(
          [[-1, 3, 3],    # "d" == unknown, eos eos
           [-1, -1, 0]],  # "f" == unknown, "e" == unknown, a
          source_v)
      self.assertAllEqual([1, 3], seq_len_v)

      with self.assertRaisesOpError("End of sequence"):
        sess.run((source, seq_len))

 def _label_ids(self, labels):
   """Converts labels to integer id space."""
   if self._label_vocabulary is None:
     if not labels.dtype.is_integer:
       raise ValueError('Labels dtype should be integer '
                        'Instead got %s.' % labels.dtype)
     label_ids = labels
   else:
     if labels.dtype != dtypes.string:
       raise ValueError('Labels dtype should be string if there is a '
                        'vocabulary. Instead got {}'.format(labels.dtype))
     label_ids = lookup_ops.index_table_from_tensor(
         vocabulary_list=tuple(self._label_vocabulary),
         name='class_id_lookup').lookup(labels)
   assert_less = check_ops.assert_less(
       label_ids,
       ops.convert_to_tensor(self._n_classes, dtype=label_ids.dtype),
       message='Label IDs must < n_classes')
   assert_greater = check_ops.assert_non_negative(
       label_ids, message='Label Ids must >= 0')
   with ops.control_dependencies((assert_less, assert_greater)):
     return array_ops.identity(label_ids)