Python string.decode函数代码示例

 def encode(self, string):
     clean_sentence_unwantedchars= '["\t\n ]+'
     string = string.encode('utf8')
     string = string.decode('utf-8')    
     string = re.sub(clean_sentence_unwantedchars, ' ', string)
     string = string.encode('ascii', 'replace').encode('utf-8')
     string = string.decode('utf-8')
     return str(string)    

def ensure_unicode(string):
    if isinstance(string, str):
        try:
            string = string.decode("utf-8")
        except UnicodeDecodeError:
            string = string.decode("unicode-escape")

    return string

def scrubstring(string):
	from scrubber import Scrubber
	scrubber = Scrubber(autolink=True)
	try:
		string = string.decode('ascii')
	except UnicodeDecodeError:
		string = string.decode('utf-8')
	string = scrubber.scrub(string)
	return string.encode('utf-8')

def ensure_unicode(stuff, encoding = 'utf8', encoding2 = 'latin-1'):
    if type(stuff) is not str and type(stuff) is not np.string_:
        return stuff
    else:
        string = stuff
    try:
        string = string.decode(encoding)
    except:
        string = string.decode(encoding2, errors = 'ignore')
    return string

def unicode_cleaner(string):
    if isinstance(string, unicode):
        return string

    try:
        return string.decode('utf-8')
    except UnicodeError:
        try:
            return string.decode('latin-1')
        except UnicodeError:
            return string.decode('utf-8', 'ignore')

    def decode_guess(self, string, encoding):
        # text is not valid utf-8, try to make sense of it
        if encoding:
            try:
                return string.decode(encoding).encode('utf-8')
            except UnicodeDecodeError:
                pass

        try:
            return string.decode('latin-1').encode('utf-8')
        except UnicodeDecodeError:
            return string.decode('ascii', 'replace').encode('utf-8')

def tryDecode(string):
	try:
		string_d = string.decode('utf-8','strict')
		enc = 'utf-8'
	except:
		try:
			string_d = string.decode(config.encoding,'strict')
			enc = config.encoding
		except:
			enc = 'iso-8859-1'
			try:
				string_d = string.decode('iso-8859-1','strict')
			except:
				string_d = string.decode('iso-8859-1','replace')
	return [string_d, enc]

	def _decode_string(self, string):
		for encoding in ['ascii', 'UTF8', 'latin-1']:
			try:
				return string.decode(encoding)
			except:
				pass
		return 'INVALID ENCODING'

def htmlenties2txt(string, encoding="latin-1"):
    """
    Resolves all the HTML entities in the input string.
    Returns a Unicode string with the entities resolved.
    """
    try:
        string = string.decode(encoding)
    except:
        pass

    i = 0
    while i < len(string):
        amp = string.find("&", i) # find & as start of entity
        if amp == -1: # not found
            break
        i = amp + 1

        semicolon = string.find(";", amp) # find ; as end of entity
        if string[amp + 1] == "#": # numerical entity like "&#039;"
            entity = string[amp:semicolon+1]
            replacement = unichr(int(entity[2:-1]))
        else:
            entity = string[amp:semicolon + 1]
            if semicolon - amp > 7:
                continue
            try:
                # the array has mappings like "Uuml" -> "�"
                replacement = unichr(htmlentitydefs.name2codepoint[entity[1:-1]])
            except KeyError:
                continue        
        string = string.replace(entity, replacement)
    return string

def string_decode(string):
    '''
    For cross compatibility between Python 2 and Python 3 strings.
    '''
    if PY_MAJOR_VERSION > 2:
        return bytes(string, 'utf-8').decode('unicode_escape')
    else:
        return string.decode('string_escape')

def tryDecode(string):
    codec = ('ascii', 'latin-1', 'utf-8')
    for c in codec:
        try:
            return string.decode(c)
        except UnicodeDecodeError:
            continue
    raise UnicodeDecodeError(string)

def string_decode(string):
    """
    For cross compatibility between Python 2 and Python 3 strings.
    """
    if PY_MAJOR_VERSION > 2:
        return bytes(string, "utf-8").decode("unicode_escape")
    else:
        return string.decode("string_escape")

def string_decode(string):
	'''
	For cross compatibility between Python 2 and Python 3 strings.
	'''
	if sys.version_info.major > 2:
		return bytes(string, 'utf-8').decode('unicode_escape')
	else:
		return string.decode('string_escape')

def mb_code(string, coding="utf-8"):
    if isinstance(string, unicode):
        return string.encode(coding)
    for c in ('utf-8', 'gb2312', 'gbk', 'gb18030', 'big5'):
        try:
            return string.decode(c).encode(coding)
        except:
            pass
    return string

def encoded(string, encoding='utf8'):
    """Cast string to binary_type.

    :param string: six.binary_type or six.text_type
    :param encoding: encoding which the object is forced to
    :return: six.binary_type
    """
    assert isinstance(string, string_types) or isinstance(string, binary_type)
    if isinstance(string, text_type):
        return string.encode(encoding)
    try:
        # make sure the string can be decoded in the specified encoding ...
        string.decode(encoding)
        return string
    except UnicodeDecodeError:
        # ... if not use latin1 as best guess to decode the string before encoding as
        # specified.
        return string.decode('latin1').encode(encoding)