Python string_ops.substr方法代码示例

# 需要导入模块: from tensorflow.python.ops import string_ops [as 别名]
# 或者: from tensorflow.python.ops.string_ops import substr [as 别名]
def getTokenWord(self, text, token_starts, token_ends):
    def _FindSubstr(input_tensor):
      text, token_start, token_length = input_tensor
      return string_ops.substr(text, token_start, token_length)

    token_lengths = token_ends - token_starts
    token_word = ragged_map_ops.map_fn(
        _FindSubstr, (text, token_starts, token_lengths),
        dtype=ragged_tensor.RaggedTensorType(
            dtype=dtypes.string, ragged_rank=1),
        infer_shape=False)
    return token_word 

# 需要导入模块: from tensorflow.python.ops import string_ops [as 别名]
# 或者: from tensorflow.python.ops.string_ops import substr [as 别名]
def decode_image(contents, channels=None, name=None):
  """Convenience function for `decode_gif`, `decode_jpeg`, and `decode_png`.
  Detects whether an image is a GIF, JPEG, or PNG, and performs the appropriate 
  operation to convert the input bytes `string` into a `Tensor` of type `uint8`.

  Note: `decode_gif` returns a 4-D array `[num_frames, height, width, 3]`, as 
  opposed to `decode_jpeg` and `decode_png`, which return 3-D arrays 
  `[height, width, num_channels]`. Make sure to take this into account when 
  constructing your graph if you are intermixing GIF files with JPEG and/or PNG 
  files.

  Args:
    contents: 0-D `string`. The encoded image bytes.
    channels: An optional `int`. Defaults to `0`. Number of color channels for 
      the decoded image.
    name: A name for the operation (optional)
    
  Returns:
    `Tensor` with type `uint8` with shape `[height, width, num_channels]` for 
      JPEG and PNG images and shape `[num_frames, height, width, 3]` for GIF 
      images.
  """
  with ops.name_scope(name, 'decode_image') as scope:
    if channels not in (None, 0, 1, 3):
      raise ValueError('channels must be in (None, 0, 1, 3)')
    substr = string_ops.substr(contents, 0, 4)

    def _gif():
      # Create assert op to check that bytes are GIF decodable
      is_gif = math_ops.equal(substr, b'\x47\x49\x46\x38', name='is_gif')
      decode_msg = 'Unable to decode bytes as JPEG, PNG, or GIF'
      assert_decode = control_flow_ops.Assert(is_gif, [decode_msg])
      # Create assert to make sure that channels is not set to 1
      # Already checked above that channels is in (None, 0, 1, 3)
      gif_channels = 0 if channels is None else channels
      good_channels = math_ops.not_equal(gif_channels, 1, name='check_channels')
      channels_msg = 'Channels must be in (None, 0, 3) when decoding GIF images'
      assert_channels = control_flow_ops.Assert(good_channels, [channels_msg])
      with ops.control_dependencies([assert_decode, assert_channels]):
        return gen_image_ops.decode_gif(contents)

    def _png():
      return gen_image_ops.decode_png(contents, channels)

    def check_png():
      is_png = math_ops.equal(substr, b'\211PNG', name='is_png')
      return control_flow_ops.cond(is_png, _png, _gif, name='cond_png')

    def _jpeg():
      return gen_image_ops.decode_jpeg(contents, channels)

    is_jpeg = math_ops.equal(substr, b'\xff\xd8\xff\xe0', name='is_jpeg')
    return control_flow_ops.cond(is_jpeg, _jpeg, check_png, name='cond_jpeg') 

# 需要导入模块: from tensorflow.python.ops import string_ops [as 别名]
# 或者: from tensorflow.python.ops.string_ops import substr [as 别名]
def break_sentences_with_offsets(self, doc):
    """Splits `doc` into sentence fragments, returns text, start & end offsets.

    Example:
                      1                  1         2         3
            012345678901234    01234567890123456789012345678901234567
      doc: 'Hello...foo bar', 'Welcome to the U.S. don't be surprised'

      fragment_text: [['Hello...', 'foo bar'], ['Welcome to the U.S.' , 'don't
      be surprised']]
      start: [[0, 8],[0, 20]]
      end: [[8, 15],[19, 38]]

    Args:
      doc: A string `Tensor` of shape [batch] with a batch of documents.

    Returns:
      A tuple of (fragment_text, start, end) where:

      fragment_text: A string `RaggedTensor` of shape [batch, (num_sentences)]
      with each input broken up into its constituent sentence fragments.
      start: A int64 `RaggedTensor` of shape [batch, (num_sentences)]
        where each entry is the inclusive beginning byte offset of a sentence.
      end: A int64 `RaggedTensor` of shape [batch, (num_sentences)]
        where each entry is the exclusive ending byte offset of a sentence.
    """
    if not isinstance(doc, ragged_tensor.RaggedTensor):
      doc = ragged_tensor.RaggedTensor.from_tensor(doc)

    # Run sentence fragmenter op v2
    fragment = gen_state_based_sentence_breaker_op.sentence_fragments_v2(
        doc=doc.flat_values)
    start, end, properties, terminal_punc_token, row_lengths = fragment

    # Pack and create `RaggedTensor`s
    start, end, properties, terminal_punc_token = tuple(
        ragged_tensor.RaggedTensor.from_row_lengths(value, row_lengths)
        for value in [start, end, properties, terminal_punc_token])

    # Helper for use within map_fn (function must only take in one argument)
    def _substring(x):
      s, pos, length = x
      return string_ops.substr(s, pos, length)

    # Extract fragment text using offsets
    fragment_text = ragged_map_ops.map_fn(
        _substring, (doc, start, math_ops.subtract(end, start)),
        infer_shape=False,
        dtype=dtypes.string)

    return fragment_text, start, end