Python codecs.lookup方法代码示例

# 需要导入模块: import codecs [as 别名]
# 或者: from codecs import lookup [as 别名]
def test_not_ascii():  # NOCOV
    """
    Make sure that the systems preferred encoding is not `ascii`.

    Otherwise `click` is raising a RuntimeError for Python3. For a detailed
    description of this very problem please consult the following gist:
    https://gist.github.com/hackebrot/937245251887197ef542

    This test also checks that `tox.ini` explicitly copies the according
    system environment variables to the test environments.
    """
    try:
        preferred_encoding = locale.getpreferredencoding()
        fs_enc = codecs.lookup(preferred_encoding).name
    except Exception:
        fs_enc = "ascii"
    assert fs_enc != "ascii" 

# 需要导入模块: import codecs [as 别名]
# 或者: from codecs import lookup [as 别名]
def __new__(cls, text, context, encoding, category, error=None):
        """Allow defaults."""

        encoding = PYTHON_ENCODING_NAMES.get(encoding, encoding).lower()

        if encoding == 'utf-8-sig':
            encoding = 'utf-8'
        if encoding.startswith('utf-16'):
            encoding = 'utf-16'
        elif encoding.startswith('utf-32'):
            encoding = 'utf-32'

        if encoding:
            encoding = codecs.lookup(encoding).name

        if RE_CATEGORY_NAME.match(category) is None and error is None:
            raise ValueError('Invalid category name in SourceText!')

        return super().__new__(cls, text, context, encoding, category, error) 

# 需要导入模块: import codecs [as 别名]
# 或者: from codecs import lookup [as 别名]
def setup(self):
        """Setup."""

        self.normalize = self.config['normalize'].upper()
        self.convert_encoding = self.config['convert_encoding'].lower()
        self.errors = self.config['errors'].lower()

        if self.convert_encoding:
            self.convert_encoding = codecs.lookup(
                filters.PYTHON_ENCODING_NAMES.get(self.default_encoding, self.default_encoding).lower()
            ).name

            # Don't generate content with BOMs
            if (
                self.convert_encoding.startswith(('utf-32', 'utf-16')) and
                not self.convert_encoding.endswith(('le', 'be'))
            ):
                self.convert_encoding += '-le'

            if self.convert_encoding == 'utf-8-sig':
                self.convert_encoding = 'utf-8' 

# 需要导入模块: import codecs [as 别名]
# 或者: from codecs import lookup [as 别名]
def get_encoding_name(self, name):
        """Get encoding name."""

        name = codecs.lookup(
            filters.PYTHON_ENCODING_NAMES.get(name, name).lower()
        ).name

        if name.startswith(('utf-32', 'utf-16')):
            name = name[:6]
            if CURRENT_ENDIAN == BIG_ENDIAN:
                name += '-be'
            else:
                name += '-le'

        if name == 'utf-8-sig':
            name = 'utf-8'
        return name 

# 需要导入模块: import codecs [as 别名]
# 或者: from codecs import lookup [as 别名]
def _populate_class_variables():
        lookup = {}
        reverse_lookup = {}
        characters_for_re = []
        for codepoint, name in list(codepoint2name.items()):
            character = unichr(codepoint)
            if codepoint != 34:
                # There's no point in turning the quotation mark into
                # ", unless it happens within an attribute value, which
                # is handled elsewhere.
                characters_for_re.append(character)
                lookup[character] = name
            # But we do want to turn " into the quotation mark.
            reverse_lookup[name] = character
        re_definition = "[%s]" % "".join(characters_for_re)
        return lookup, reverse_lookup, re.compile(re_definition) 

# 需要导入模块: import codecs [as 别名]
# 或者: from codecs import lookup [as 别名]
def test_bad_stream_exception(all_parsers, csv_dir_path):
    # see gh-13652
    #
    # This test validates that both the Python engine and C engine will
    # raise UnicodeDecodeError instead of C engine raising ParserError
    # and swallowing the exception that caused read to fail.
    path = os.path.join(csv_dir_path, "sauron.SHIFT_JIS.csv")
    codec = codecs.lookup("utf-8")
    utf8 = codecs.lookup('utf-8')
    parser = all_parsers

    msg = ("'utf-8' codec can't decode byte" if compat.PY3
           else "'utf8' codec can't decode byte")

    # Stream must be binary UTF8.
    with open(path, "rb") as handle, codecs.StreamRecoder(
            handle, utf8.encode, utf8.decode, codec.streamreader,
            codec.streamwriter) as stream:

        with pytest.raises(UnicodeDecodeError, match=msg):
            parser.read_csv(stream) 

# 需要导入模块: import codecs [as 别名]
# 或者: from codecs import lookup [as 别名]
def _get_encoding(encoding_or_label):
    """
    Accept either an encoding object or label.

    :param encoding: An :class:`Encoding` object or a label string.
    :returns: An :class:`Encoding` object.
    :raises: :exc:`~exceptions.LookupError` for an unknown label.

    """
    if hasattr(encoding_or_label, 'codec_info'):
        return encoding_or_label

    encoding = lookup(encoding_or_label)
    if encoding is None:
        raise LookupError('Unknown encoding label: %r' % encoding_or_label)
    return encoding 

# 需要导入模块: import codecs [as 别名]
# 或者: from codecs import lookup [as 别名]
def _set_iptc_charset(self, charset):
        if charset is None:
            self._del_iptc_charset()
            return

        try:
            name = codecs.lookup(charset).name
        except LookupError as error:
            raise ValueError(error)

        else:
            charsets = {'utf-8': '\x1b%G'}
            try:
                self['Iptc.Envelope.CharacterSet'] = (charsets[name],)
            except KeyError:
                raise ValueError('Unhandled charset: %s' % name) 

# 需要导入模块: import codecs [as 别名]
# 或者: from codecs import lookup [as 别名]
def normalize_codec_name(chardet_name):
    """
    Normalizes chardet codec names to Python codec names.
    :param chardet_name: chardet codec names
    :return: Python codec names. See: https://docs.python.org/3.7/library/codecs.html#standard-encodings
    """

    python_name = chardet_name.lower().replace('iso-', 'iso').replace('-', '_')
    python_name = codecs.lookup(python_name).name

    # Since chardet only recognized all GB-based target_encoding as 'gb2312', the decoding will fail when the text file
    # contains certain special charaters. To make it more special-character-tolerant, we should
    # upgrade the target_encoding to 'gb18030', which is a character set larger than gb2312.
    if python_name == 'gb2312':
        return 'gb18030'

    return python_name 

# 需要导入模块: import codecs [as 别名]
# 或者: from codecs import lookup [as 别名]
def test_all(self):
        api = (
            "encode", "decode",
            "register", "CodecInfo", "Codec", "IncrementalEncoder",
            "IncrementalDecoder", "StreamReader", "StreamWriter", "lookup",
            "getencoder", "getdecoder", "getincrementalencoder",
            "getincrementaldecoder", "getreader", "getwriter",
            "register_error", "lookup_error",
            "strict_errors", "replace_errors", "ignore_errors",
            "xmlcharrefreplace_errors", "backslashreplace_errors",
            "open", "EncodedFile",
            "iterencode", "iterdecode",
            "BOM", "BOM_BE", "BOM_LE",
            "BOM_UTF8", "BOM_UTF16", "BOM_UTF16_BE", "BOM_UTF16_LE",
            "BOM_UTF32", "BOM_UTF32_BE", "BOM_UTF32_LE",
            "BOM32_BE", "BOM32_LE", "BOM64_BE", "BOM64_LE",  # Undocumented
            "StreamReaderWriter", "StreamRecoder",
        )
        self.assertEqual(sorted(api), sorted(codecs.__all__))
        for api in codecs.__all__:
            getattr(codecs, api) 

# 需要导入模块: import codecs [as 别名]
# 或者: from codecs import lookup [as 别名]
def test_codecs_lookup(self):
        l = []
        def my_func(encoding, cache = l):
            l.append(encoding)
        
        codecs.register(my_func)
        allchars = ''.join([chr(i) for i in xrange(1, 256)])
        try:
            codecs.lookup(allchars)
            self.assertUnreachable()
        except LookupError:
            pass
            
        lowerchars = allchars.lower().replace(' ', '-')
        for i in xrange(1, 255):
            if l[0][i] != lowerchars[i]:
                self.assertTrue(False, 'bad chars at index %d: %r %r' % (i, l[0][i], lowerchars[i]))
                
        self.assertRaises(TypeError, codecs.lookup, '\0')
        self.assertRaises(TypeError, codecs.lookup, 'abc\0')
        self.assertEqual(len(l), 1) 

# 需要导入模块: import codecs [as 别名]
# 或者: from codecs import lookup [as 别名]
def _hack_lookup_codecs():
  import encodings
  import pkgutil
  import codecs
  for _, name, _ in pkgutil.iter_modules(encodings.__path__):
    if name in ('aliases', 'mbcs'):
      continue
    codecs.lookup(name) 

# 需要导入模块: import codecs [as 别名]
# 或者: from codecs import lookup [as 别名]
def to_py(self, value: _StrUnset) -> _StrUnsetNone:
        self._basic_py_validation(value, str)
        if isinstance(value, usertypes.Unset):
            return value
        elif not value:
            return None
        try:
            codecs.lookup(value)
        except LookupError:
            raise configexc.ValidationError(value, "is not a valid encoding!")
        return value 

# 需要导入模块: import codecs [as 别名]
# 或者: from codecs import lookup [as 别名]
def _walk_src(self, targets, flags, limit, pipeline, expect_match):
        """Walk source and parse files."""

        found_something = False
        for target in targets:
            # Glob using `S` for patterns wit `|` and `O` to exclude directories.
            kwargs = {"flags": flags | glob.S | glob.O}
            kwargs['limit'] = limit
            for f in glob.iglob(target, **kwargs):
                found_something = True
                self.log('', 2)
                self.log('> Processing: %s' % f, 1)
                if pipeline:
                    try:
                        yield pipeline[0]._run_first(f)
                    except Exception as e:
                        err = self.get_error(e)
                        yield [filters.SourceText('', f, '', '', err)]
                else:
                    try:
                        if self.default_encoding:
                            encoding = filters.PYTHON_ENCODING_NAMES.get(
                                self.default_encoding, self.default_encoding
                            ).lower()
                            encoding = codecs.lookup(encoding).name
                        else:
                            encoding = self.default_encoding
                        yield [filters.SourceText('', f, encoding, 'file')]
                    except Exception as e:
                        err = self.get_error(e)
                        yield [filters.SourceText('', f, '', '', err)]
        if not found_something and expect_match:
            raise RuntimeError(
                'None of the source targets from the configuration match any files:\n{}'.format(
                    '\n'.join('- {}'.format(target) for target in targets)
                )
            ) 

# 需要导入模块: import codecs [as 别名]
# 或者: from codecs import lookup [as 别名]
def _codec(self, charset):
        if not charset:
            return charset
        codec = None
        try:
            codecs.lookup(charset)
            codec = charset
        except (LookupError, ValueError):
            pass
        return codec


    # A partial mapping of ISO-Latin-1 to HTML entities/XML numeric entities.