Python six.ensure_text方法代码示例

# 需要导入模块: import six [as 别名]
# 或者: from six import ensure_text [as 别名]
def preprocess_text(inputs, remove_space=True, lower=False):
    """preprocess data by removing extra space and normalize data."""
    outputs = inputs
    if remove_space:
        outputs = " ".join(inputs.strip().split())

    if six.PY2 and isinstance(outputs, str):
        try:
            outputs = six.ensure_text(outputs, "utf-8")
        except UnicodeDecodeError:
            outputs = six.ensure_text(outputs, "latin-1")

    outputs = unicodedata.normalize("NFKD", outputs)
    outputs = "".join([c for c in outputs if not unicodedata.combining(c)])
    if lower:
        outputs = outputs.lower()

    return outputs 

# 需要导入模块: import six [as 别名]
# 或者: from six import ensure_text [as 别名]
def convert_to_unicode(text):
    """Converts `text` to Unicode (if it's not already), assuming utf-8 input."""
    if six.PY3:
        if isinstance(text, str):
            return text
        elif isinstance(text, bytes):
            return six.ensure_text(text, "utf-8", "ignore")
        else:
            raise ValueError("Unsupported string type: %s" % (type(text)))
    elif six.PY2:
        if isinstance(text, str):
            return six.ensure_text(text, "utf-8", "ignore")
        elif isinstance(text, six.text_type):
            return text
        else:
            raise ValueError("Unsupported string type: %s" % (type(text)))
    else:
        raise ValueError("Not running on Python2 or Python 3?") 

# 需要导入模块: import six [as 别名]
# 或者: from six import ensure_text [as 别名]
def printable_text(text):
    """Returns text encoded in a way suitable for print or `tf.logging`."""

    # These functions want `str` for both Python2 and Python3, but in one case
    # it's a Unicode string and in the other it's a byte string.
    if six.PY3:
        if isinstance(text, str):
            return text
        elif isinstance(text, bytes):
            return six.ensure_text(text, "utf-8", "ignore")
        else:
            raise ValueError("Unsupported string type: %s" % (type(text)))
    elif six.PY2:
        if isinstance(text, str):
            return text
        elif isinstance(text, six.text_type):
            return six.ensure_binary(text, "utf-8")
        else:
            raise ValueError("Unsupported string type: %s" % (type(text)))
    else:
        raise ValueError("Not running on Python2 or Python 3?") 

# 需要导入模块: import six [as 别名]
# 或者: from six import ensure_text [as 别名]
def _CustomShortDebugString(tf_example):
  text = []
  for name, value in sorted(six.iteritems(tf_example.features.feature)):
    if value.HasField('bytes_list'):
      if FLAGS.bytes_as_utf8:
        utf8_values = [
            six.ensure_text(v, 'utf-8') for v in value.bytes_list.value
        ]
        value_string = _ListDebugString(utf8_values)
      else:
        value_string = _ListDebugString(value.bytes_list.value)
    elif value.HasField('float_list'):
      value_string = _ListDebugString(value.float_list.value)
    elif value.HasField('int64_list'):
      value_string = _ListDebugString(value.int64_list.value, to_string=repr)
    text += ['%s: %s' % (name, value_string)]
  return '\n'.join(text) 

# 需要导入模块: import six [as 别名]
# 或者: from six import ensure_text [as 别名]
def __init__(self, wpm_filepath, merge_prob=1.):
    """Create a WPM encoder.

    Args:
      wpm_filepath: a path to the file containing the vocabulary.
      merge_prob: the probability of merging tokens while encoding.
    """
    # Load vocabulary file.
    lines = py_utils.ReadFileLines(wpm_filepath)

    self._pieces = []
    for line in lines:
      if isinstance(line, six.binary_type):
        line = six.ensure_text(line, 'utf-8')
      piece = line.strip().split('\t')[0]
      self._pieces.append(piece)
    self._merge_prob = merge_prob 

# 需要导入模块: import six [as 别名]
# 或者: from six import ensure_text [as 别名]
def normalize_path(path):
    # type: (AnyStr) -> AnyStr
    """
    Return a case-normalized absolute variable-expanded path.

    :param str path: The non-normalized path
    :return: A normalized, expanded, case-normalized path
    :rtype: str
    """

    path = os.path.abspath(os.path.expandvars(os.path.expanduser(str(path))))
    if os.name == "nt" and os.path.exists(path):
        try:
            from ctypes import create_unicode_buffer, windll
        except ImportError:
            path = os.path.normpath(os.path.normcase(path))
        else:
            BUFSIZE = 500
            buffer = create_unicode_buffer(BUFSIZE)
            get_long_path_name = windll.kernel32.GetLongPathNameW
            get_long_path_name(six.ensure_text(path), buffer, BUFSIZE)
            path = buffer.value
        return path

    return os.path.normpath(os.path.normcase(path)) 

# 需要导入模块: import six [as 别名]
# 或者: from six import ensure_text [as 别名]
def preprocess_text(inputs, remove_space=True, lower=False):
  """preprocess data by removing extra space and normalize data."""
  outputs = inputs
  if remove_space:
    outputs = " ".join(inputs.strip().split())

  if six.PY2 and isinstance(outputs, str):
    try:
      outputs = six.ensure_text(outputs, "utf-8")
    except UnicodeDecodeError:
      outputs = six.ensure_text(outputs, "latin-1")

  outputs = unicodedata.normalize("NFKD", outputs)
  outputs = "".join([c for c in outputs if not unicodedata.combining(c)])
  if lower:
    outputs = outputs.lower()

  return outputs 

# 需要导入模块: import six [as 别名]
# 或者: from six import ensure_text [as 别名]
def convert_to_unicode(text):
  """Converts `text` to Unicode (if it's not already), assuming utf-8 input."""
  if six.PY3:
    if isinstance(text, str):
      return text
    elif isinstance(text, bytes):
      return six.ensure_text(text, "utf-8", "ignore")
    else:
      raise ValueError("Unsupported string type: %s" % (type(text)))
  elif six.PY2:
    if isinstance(text, str):
      return six.ensure_text(text, "utf-8", "ignore")
    elif isinstance(text, six.text_type):
      return text
    else:
      raise ValueError("Unsupported string type: %s" % (type(text)))
  else:
    raise ValueError("Not running on Python2 or Python 3?") 

# 需要导入模块: import six [as 别名]
# 或者: from six import ensure_text [as 别名]
def printable_text(text):
  """Returns text encoded in a way suitable for print or `tf.logging`."""

  # These functions want `str` for both Python2 and Python3, but in one case
  # it's a Unicode string and in the other it's a byte string.
  if six.PY3:
    if isinstance(text, str):
      return text
    elif isinstance(text, bytes):
      return six.ensure_text(text, "utf-8", "ignore")
    else:
      raise ValueError("Unsupported string type: %s" % (type(text)))
  elif six.PY2:
    if isinstance(text, str):
      return text
    elif isinstance(text, six.text_type):
      return six.ensure_binary(text, "utf-8")
    else:
      raise ValueError("Unsupported string type: %s" % (type(text)))
  else:
    raise ValueError("Not running on Python2 or Python 3?") 

# 需要导入模块: import six [as 别名]
# 或者: from six import ensure_text [as 别名]
def create_example(filename, sample_rate, load_audio_with_librosa):
  """Processes an audio file into an Example proto."""
  wav_data = tf.gfile.Open(filename, 'rb').read()
  example_list = list(
      audio_label_data_utils.process_record(
          wav_data=wav_data,
          sample_rate=sample_rate,
          ns=music_pb2.NoteSequence(),
          # decode to handle filenames with extended characters.
          example_id=six.ensure_text(filename, 'utf-8'),
          min_length=0,
          max_length=-1,
          allow_empty_notesequence=True,
          load_audio_with_librosa=load_audio_with_librosa))
  assert len(example_list) == 1
  return example_list[0].SerializeToString() 

# 需要导入模块: import six [as 别名]
# 或者: from six import ensure_text [as 别名]
def _test_json_encode(benchmark, json_lib, log_tuple):
    """
    :param json_lib: JSON library to use.
    :param log_tuple: Tuple with (log_filename, log_bytes_to_use).
    """
    set_json_lib(json_lib)

    file_name, bytes_to_read = log_tuple

    if log_tuple not in CACHED_TEST_DATA["encode"]:
        data = read_bytes_from_log_fixture_file(file_name, bytes_to_read)
        data = six.ensure_text(data)

        CACHED_TEST_DATA["encode"][log_tuple] = data

    data = CACHED_TEST_DATA["encode"][log_tuple]

    def run_benchmark():
        return json_encode(data)

    result = benchmark.pedantic(run_benchmark, iterations=20, rounds=50)

    assert get_json_lib() == json_lib
    assert isinstance(result, six.text_type)
    # assert json.dumps(data) == result 

# 需要导入模块: import six [as 别名]
# 或者: from six import ensure_text [as 别名]
def json_encode(obj, output=None, binary=False):
    """Encodes an object into a JSON string.

    @param obj: The object to serialize
    @param output: If not None, a file-like object to which the serialization should be written.
    @param binary: If True return binary string, otherwise text string.
    @type obj: dict|list|six.text_type
    @type binary: bool
    """
    # 2->TODO encode json according to 'binary' flag.
    if binary:

        result = six.ensure_binary(_json_encode(obj, None))
        if output:
            output.write(result)
        else:
            return result
    else:
        return six.ensure_text(_json_encode(obj, output)) 

# 需要导入模块: import six [as 别名]
# 或者: from six import ensure_text [as 别名]
def formatException(self, ei):
        # We just want to indent the stack trace to make it easier to write a parsing rule to detect it.
        output = io.StringIO()
        try:
            # 2->TODO 'logging.Formatter.formatException' returns binary data (str) in python2,
            #  so it will not work with io.StringIO here.
            exception_string = six.ensure_text(
                logging.Formatter.formatException(self, ei)
            )

            for line in exception_string.splitlines(True):
                output.write("  ")
                output.write(line)
            return output.getvalue()
        finally:
            output.close() 

# 需要导入模块: import six [as 别名]
# 或者: from six import ensure_text [as 别名]
def _get_evaluation_result(self, request):
    run = request.args.get('run')
    try:
      run = six.ensure_text(run)
    except (UnicodeDecodeError, AttributeError):
      pass

    data = []
    try:
      eval_result_output_dir = six.ensure_text(
          self._multiplexer.Tensors(run, FairnessIndicatorsPlugin.plugin_name)
          [0].tensor_proto.string_val[0])
      eval_result = tfma.load_eval_result(output_path=eval_result_output_dir)
      # TODO(b/141283811): Allow users to choose different model output names
      # and class keys in case of multi-output and multi-class model.
      data = widget_view.convert_slicing_metrics_to_ui_input(
          eval_result.slicing_metrics)
    except (KeyError, json_format.ParseError) as error:
      logging.info('Error while fetching evaluation data, %s', error)
    return http_util.Respond(request, data, content_type='application/json') 

# 需要导入模块: import six [as 别名]
# 或者: from six import ensure_text [as 别名]
def _get_evaluation_result_from_remote_path(self, request):
    evaluation_output_path = request.args.get('evaluation_output_path')
    try:
      evaluation_output_path = six.ensure_text(evaluation_output_path)
    except (UnicodeDecodeError, AttributeError):
      pass
    try:
      eval_result = tfma.load_eval_result(
          os.path.dirname(evaluation_output_path),
          output_file_format=self._get_output_file_format(
              evaluation_output_path))
      data = widget_view.convert_slicing_metrics_to_ui_input(
          eval_result.slicing_metrics)
    except (KeyError, json_format.ParseError) as error:
      logging.info('Error while fetching evaluation data, %s', error)
      data = []
    return http_util.Respond(request, data, content_type='application/json')