Python encodings.normalize_encoding方法代码示例

# 需要导入模块: import encodings [as 别名]
# 或者: from encodings import normalize_encoding [as 别名]
def decodeURLContent(content):
    """Decodes the content read from a URL"""
    project = GlobalData().project
    if project.isLoaded():
        projectEncoding = project.props['encoding']
        if projectEncoding:
            if not isValidEncoding(projectEncoding):
                raise Exception(
                    "The prject encoding " + projectEncoding + " is invalid. "
                    "Please select a valid one in the project properties and "
                    "try again.")
            return content.decode(
                encodings.normalize_encoding(projectEncoding))

    # Check the IDE wide encoding
    ideEncoding = Settings()['encoding']
    if ideEncoding:
        if not isValidEncoding(ideEncoding):
            raise Exception("The ide encoding " + ideEncoding + " is invalid. "
                            "Please set a valid one and try again.")
        return content.decode(encodings.normalize_encoding(ideEncoding))

    # The default one
    return content.decode(DEFAULT_ENCODING) 

# 需要导入模块: import encodings [as 别名]
# 或者: from encodings import normalize_encoding [as 别名]
def _replace_encoding(code, encoding):
    if '.' in code:
        langname = code[:code.index('.')]
    else:
        langname = code
    # Convert the encoding to a C lib compatible encoding string
    norm_encoding = encodings.normalize_encoding(encoding)
    #print('norm encoding: %r' % norm_encoding)
    norm_encoding = encodings.aliases.aliases.get(norm_encoding.lower(),
                                                  norm_encoding)
    #print('aliased encoding: %r' % norm_encoding)
    encoding = norm_encoding
    norm_encoding = norm_encoding.lower()
    if norm_encoding in locale_encoding_alias:
        encoding = locale_encoding_alias[norm_encoding]
    else:
        norm_encoding = norm_encoding.replace('_', '')
        norm_encoding = norm_encoding.replace('-', '')
        if norm_encoding in locale_encoding_alias:
            encoding = locale_encoding_alias[norm_encoding]
    #print('found encoding %r' % encoding)
    return langname + '.' + encoding 

# 需要导入模块: import encodings [as 别名]
# 或者: from encodings import normalize_encoding [as 别名]
def get_encoding(html_string):
    """
    Performs a simple RE based parsing of a HTML page (represented as a
    string) and extracts declared character encoding from the meta tags.
    If the extraction is successful, the encoding is returned. Otherwise,
    None is returned.
    """
    re_meta1 = re.compile('''<meta\s+http-equiv=['"]?content-type['"]?\s+content=['"]?[^'"]*charset=([^'"]+)''', re.I)
    re_meta2 = re.compile('''<meta\s+content=['"]?[^'"]*charset=([^'"]+)['"]?\s+http-equiv=['"]?content-type['"]?''', re.I)
    re_meta3 = re.compile('''<meta\s+http-equiv=['"]?charset['"]?\s+content=['"]?([^'"]+)''', re.I)
    re_meta4 = re.compile('''<meta\s+content=['"]?([^'"]+)['"]?\s+http-equiv=['"]?charset['"]?''', re.I)
    re_meta5 = re.compile('''<meta\s+charset=['"]?([^'"]+)''', re.I)
    for re_meta in (re_meta1, re_meta2, re_meta3, re_meta4, re_meta5):
        m = re_meta.search(html_string)
        if m:
            meta_encoding = m.group(1)
            return normalize_encoding(meta_encoding)
    return None 

# 需要导入模块: import encodings [as 别名]
# 或者: from encodings import normalize_encoding [as 别名]
def setUp(self):
        # There's no way to unregister a codec search function, so we just
        # ensure we render this one fairly harmless after the test
        # case finishes by using the test case repr as the codec name
        # The codecs module normalizes codec names, although this doesn't
        # appear to be formally documented...
        # We also make sure we use a truly unique id for the custom codec
        # to avoid issues with the codec cache when running these tests
        # multiple times (e.g. when hunting for refleaks)
        unique_id = repr(self) + str(id(self))
        self.codec_name = encodings.normalize_encoding(unique_id).lower()

        # We store the object to raise on the instance because of a bad
        # interaction between the codec caching (which means we can't
        # recreate the codec entry) and regrtest refleak hunting (which
        # runs the same test instance multiple times). This means we
        # need to ensure the codecs call back in to the instance to find
        # out which exception to raise rather than binding them in a
        # closure to an object that may change on the next run
        self.obj_to_raise = RuntimeError 

# 需要导入模块: import encodings [as 别名]
# 或者: from encodings import normalize_encoding [as 别名]
def search_function(name):
	name = encodings.normalize_encoding(name) # Rather undocumented...
	if name in _extended_encodings:
		if name not in _cache:
			base_encoding, mapping = _extended_encodings[name]
			assert(name[-4:] == "_ttx")
			# Python 2 didn't have any of the encodings that we are implementing
			# in this file.  Python 3 added aliases for the East Asian ones, mapping
			# them "temporarily" to the same base encoding as us, with a comment
			# suggesting that full implementation will appear some time later.
			# As such, try the Python version of the x_mac_... first, if that is found,
			# use *that* as our base encoding.  This would make our encoding upgrade
			# to the full encoding when and if Python finally implements that.
			# http://bugs.python.org/issue24041
			base_encodings = [name[:-4], base_encoding]
			for base_encoding in base_encodings:
				try:
					codecs.lookup(base_encoding)
				except LookupError:
					continue
				_cache[name] = ExtendCodec(name, base_encoding, mapping)
				break
		return _cache[name].info

	return None 

# 需要导入模块: import encodings [as 别名]
# 或者: from encodings import normalize_encoding [as 别名]
def isValidEncoding(enc):
    """Checks if it is a valid encoding"""
    norm_enc = encodings.normalize_encoding(enc).lower()
    if norm_enc in SUPPORTED_CODECS:
        return True
    if norm_enc in [encodings.normalize_encoding(supp_enc)
                    for supp_enc in SUPPORTED_CODECS]:
        return True

    # Check the aliases as well
    if norm_enc in encodings.aliases.aliases:
        return True
    return False 

# 需要导入模块: import encodings [as 别名]
# 或者: from encodings import normalize_encoding [as 别名]
def getNormalizedEncoding(enc, validityCheck=True):
    """Returns a normalized encoding or throws an exception"""
    if validityCheck:
        if not isValidEncoding(enc):
            raise Exception('Unsupported encoding ' + enc)
    norm_enc = encodings.normalize_encoding(enc).lower()
    return encodings.aliases.aliases.get(norm_enc, norm_enc) 

# 需要导入模块: import encodings [as 别名]
# 或者: from encodings import normalize_encoding [as 别名]
def _replace_encoding(code, encoding):
    if '.' in code:
        langname = code[:code.index('.')]
    else:
        langname = code
    # Convert the encoding to a C lib compatible encoding string
    norm_encoding = encodings.normalize_encoding(encoding)
    #print('norm encoding: %r' % norm_encoding)
    norm_encoding = encodings.aliases.aliases.get(norm_encoding,
                                                  norm_encoding)
    #print('aliased encoding: %r' % norm_encoding)
    encoding = locale_encoding_alias.get(norm_encoding,
                                         norm_encoding)
    #print('found encoding %r' % encoding)
    return langname + '.' + encoding 

# 需要导入模块: import encodings [as 别名]
# 或者: from encodings import normalize_encoding [as 别名]
def normalize_encoding(encoding):
    "Returns a normalized form of the encoding."
    import encodings
    norm = encodings.normalize_encoding(encoding).lower()
    if norm in encodings.aliases.aliases.values():
        return norm
    return encodings.aliases.aliases.get(norm) 

# 需要导入模块: import encodings [as 别名]
# 或者: from encodings import normalize_encoding [as 别名]
def _c18n_encoding(encoding):
    """Cannonicalize an encoding name

    This performs normalization and translates aliases using python's
    encoding aliases
    """
    normed = encodings.normalize_encoding(encoding).lower()
    return encodings.aliases.aliases.get(normed, normed) 

# 需要导入模块: import encodings [as 别名]
# 或者: from encodings import normalize_encoding [as 别名]
def find_encodings(enc=None, system=False):
    """Find functions for encoding translations for a specific codec.

    :param str enc: The codec to find translation functions for. It will be
                    normalized by converting to lowercase, excluding
                    everything which is not ascii, and hyphens will be
                    converted to underscores.

    :param bool system: If True, find encodings based on the system's stdin
                        encoding, otherwise assume utf-8.

    :raises: :exc:LookupError if the normalized codec, ``enc``, cannot be
             found in Python's encoding translation map.
    """
    if not enc:
        enc = 'utf-8'

    if system:
        if getattr(sys.stdin, 'encoding', None) is None:
            enc = sys.stdin.encoding
            log.debug("Obtained encoding from stdin: %s" % enc)
        else:
            enc = 'ascii'

    ## have to have lowercase to work, see
    ## http://docs.python.org/dev/library/codecs.html#standard-encodings
    enc = enc.lower()
    codec_alias = encodings.normalize_encoding(enc)

    codecs.register(encodings.search_function)
    coder = codecs.lookup(codec_alias)

    return coder 

# 需要导入模块: import encodings [as 别名]
# 或者: from encodings import normalize_encoding [as 别名]
def _c18n_encoding(encoding):
    """Cannonicalize an encoding name
    This performs normalization and translates aliases using python's
    encoding aliases
    """
    normed = encodings.normalize_encoding(encoding).lower()
    return encodings.aliases.aliases.get(normed, normed)