Python codecs.iterencode方法代码示例

# 需要导入模块: import codecs [as 别名]
# 或者: from codecs import iterencode [as 别名]
def test_all(self):
        api = (
            "encode", "decode",
            "register", "CodecInfo", "Codec", "IncrementalEncoder",
            "IncrementalDecoder", "StreamReader", "StreamWriter", "lookup",
            "getencoder", "getdecoder", "getincrementalencoder",
            "getincrementaldecoder", "getreader", "getwriter",
            "register_error", "lookup_error",
            "strict_errors", "replace_errors", "ignore_errors",
            "xmlcharrefreplace_errors", "backslashreplace_errors",
            "open", "EncodedFile",
            "iterencode", "iterdecode",
            "BOM", "BOM_BE", "BOM_LE",
            "BOM_UTF8", "BOM_UTF16", "BOM_UTF16_BE", "BOM_UTF16_LE",
            "BOM_UTF32", "BOM_UTF32_BE", "BOM_UTF32_LE",
            "BOM32_BE", "BOM32_LE", "BOM64_BE", "BOM64_LE",  # Undocumented
            "StreamReaderWriter", "StreamRecoder",
        )
        self.assertEqual(sorted(api), sorted(codecs.__all__))
        for api in codecs.__all__:
            getattr(codecs, api) 

# 需要导入模块: import codecs [as 别名]
# 或者: from codecs import iterencode [as 别名]
def test_all(self):
        api = (
            "encode", "decode",
            "register", "CodecInfo", "Codec", "IncrementalEncoder",
            "IncrementalDecoder", "StreamReader", "StreamWriter", "lookup",
            "getencoder", "getdecoder", "getincrementalencoder",
            "getincrementaldecoder", "getreader", "getwriter",
            "register_error", "lookup_error",
            "strict_errors", "replace_errors", "ignore_errors",
            "xmlcharrefreplace_errors", "backslashreplace_errors",
            "namereplace_errors",
            "open", "EncodedFile",
            "iterencode", "iterdecode",
            "BOM", "BOM_BE", "BOM_LE",
            "BOM_UTF8", "BOM_UTF16", "BOM_UTF16_BE", "BOM_UTF16_LE",
            "BOM_UTF32", "BOM_UTF32_BE", "BOM_UTF32_LE",
            "BOM32_BE", "BOM32_LE", "BOM64_BE", "BOM64_LE",  # Undocumented
            "StreamReaderWriter", "StreamRecoder",
        )
        self.assertCountEqual(api, codecs.__all__)
        for api in codecs.__all__:
            getattr(codecs, api) 

# 需要导入模块: import codecs [as 别名]
# 或者: from codecs import iterencode [as 别名]
def test_all(self):
        api = (
            "encode", "decode",
            "register", "CodecInfo", "Codec", "IncrementalEncoder",
            "IncrementalDecoder", "StreamReader", "StreamWriter", "lookup",
            "getencoder", "getdecoder", "getincrementalencoder",
            "getincrementaldecoder", "getreader", "getwriter",
            "register_error", "lookup_error",
            "strict_errors", "replace_errors", "ignore_errors",
            "xmlcharrefreplace_errors", "backslashreplace_errors",
            "open", "EncodedFile",
            "iterencode", "iterdecode",
            "BOM", "BOM_BE", "BOM_LE",
            "BOM_UTF8", "BOM_UTF16", "BOM_UTF16_BE", "BOM_UTF16_LE",
            "BOM_UTF32", "BOM_UTF32_BE", "BOM_UTF32_LE",
            "BOM32_BE", "BOM32_LE", "BOM64_BE", "BOM64_LE",  # Undocumented
            "StreamReaderWriter", "StreamRecoder",
        )
        self.assertCountEqual(api, codecs.__all__)
        for api in codecs.__all__:
            getattr(codecs, api) 

# 需要导入模块: import codecs [as 别名]
# 或者: from codecs import iterencode [as 别名]
def __iter_extended_rows(self):

        # For PY2 encode/decode
        if six.PY2:
            # Reader requires utf-8 encoded stream
            bytes = iterencode(self.__chars, 'utf-8')
            sample, dialect = self.__prepare_dialect(bytes)
            items = csv.reader(chain(sample, bytes), dialect=dialect)
            for row_number, item in enumerate(items, start=1):
                values = []
                for value in item:
                    value = value.decode('utf-8')
                    values.append(value)
                yield (row_number, None, list(values))

        # For PY3 use chars
        else:
            sample, dialect = self.__prepare_dialect(self.__chars)
            items = csv.reader(chain(sample, self.__chars), dialect=dialect)
            for row_number, item in enumerate(items, start=1):
                yield (row_number, None, list(item)) 

# 需要导入模块: import codecs [as 别名]
# 或者: from codecs import iterencode [as 别名]
def encode_all(f=None, **kwargs):
    """
    Encode unicode into bytes (str)
    """
    names = kwargs.pop('fieldnames', None)
    encoding = kwargs.pop('encoding', None) if f else False

    if PY2:
        decoded = codecs.iterdecode(f, encoding) if encoding else f
        ekwargs = {encode(k): encode(v) for k, v in kwargs.items()}
    else:
        decoded, ekwargs = f, kwargs

    res = {
        'f': codecs.iterencode(decoded, ENCODING) if f and PY2 else decoded,
        'fieldnames': [encode(x) for x in names] if names and PY2 else names,
        'drkwargs': use_keys_from(ekwargs, READER_KEYS),
        'dwkwargs': use_keys_from(ekwargs, WRITER_KEYS),
        'fmtparams': use_keys_from(ekwargs, FMTKEYS)}

    return res 

# 需要导入模块: import codecs [as 别名]
# 或者: from codecs import iterencode [as 别名]
def write_encoded_data(self, f):
    # Sometimes users give us invalid utf-8 data. They shouldn't, but it does
    # happen every once and a while. Just ignore it, and replace with �.
    # We're assuming users only want to write text data out.
    # self.data can be large, so be careful to do the conversion in chunks
    # while streaming the data out, instead of requiring a full copy.
    n = 1 << 16
    # This is a generator expression, so this only copies one chunk of
    # self.data at any one time.
    chunks = (self.data[i:i + n] for i in xrange(0, len(self.data), n))
    decoded = codecs.iterdecode(chunks, 'utf-8', 'replace')
    for chunk in codecs.iterencode(decoded, 'utf-8'):
      f.write(chunk) 

# 需要导入模块: import codecs [as 别名]
# 或者: from codecs import iterencode [as 别名]
def read_decoded_data(self, f):
    # This ensures that the raw result bytes we got are, in fact, valid utf-8,
    # replacing invalid bytes with �. Because python2's unicode support is
    # wonky, we re-encode the now-valid-utf-8 back into a str object so that
    # users don't need to deal with `unicode` objects.
    # The file contents can be large, so be careful to do the conversion in
    # chunks while streaming the data in, instead of requiring a full copy.
    n = 1 << 16
    chunks = iter(lambda: f.read(n), '')
    decoded = codecs.iterdecode(chunks, 'utf-8', 'replace')
    return ''.join(codecs.iterencode(decoded, 'utf-8')) 

# 需要导入模块: import codecs [as 别名]
# 或者: from codecs import iterencode [as 别名]
def test_incremental_encode(self):
        self.assertEqual(
            "".join(codecs.iterencode(u"python.org", "idna")),
            "python.org"
        )
        self.assertEqual(
            "".join(codecs.iterencode(u"python.org.", "idna")),
            "python.org."
        )
        self.assertEqual(
            "".join(codecs.iterencode(u"pyth\xf6n.org.", "idna")),
            "xn--pythn-mua.org."
        )
        self.assertEqual(
            "".join(codecs.iterencode(u"pyth\xf6n.org.", "idna")),
            "xn--pythn-mua.org."
        )

        encoder = codecs.getincrementalencoder("idna")()
        self.assertEqual(encoder.encode(u"\xe4x"), "")
        self.assertEqual(encoder.encode(u"ample.org"), "xn--xample-9ta.")
        self.assertEqual(encoder.encode(u"", True), "org")

        encoder.reset()
        self.assertEqual(encoder.encode(u"\xe4x"), "")
        self.assertEqual(encoder.encode(u"ample.org."), "xn--xample-9ta.org.")
        self.assertEqual(encoder.encode(u"", True), "") 

# 需要导入模块: import codecs [as 别名]
# 或者: from codecs import iterencode [as 别名]
def test_basics_capi(self):
        from _testcapi import codec_incrementalencoder, codec_incrementaldecoder
        s = u"abc123"  # all codecs should be able to encode these
        for encoding in all_unicode_encodings:
            if encoding not in broken_incremental_coders:
                # check incremental decoder/encoder and iterencode()/iterdecode()
                try:
                    cencoder = codec_incrementalencoder(encoding)
                except LookupError:  # no IncrementalEncoder
                    pass
                else:
                    # check C API
                    encodedresult = ""
                    for c in s:
                        encodedresult += cencoder.encode(c)
                    encodedresult += cencoder.encode(u"", True)
                    cdecoder = codec_incrementaldecoder(encoding)
                    decodedresult = u""
                    for c in encodedresult:
                        decodedresult += cdecoder.decode(c)
                    decodedresult += cdecoder.decode("", True)
                    self.assertEqual(decodedresult, s,
                                     "encoding=%r" % encoding)

                if encoding not in only_strict_mode:
                    # check incremental decoder/encoder with errors argument
                    try:
                        cencoder = codec_incrementalencoder(encoding, "ignore")
                    except LookupError:  # no IncrementalEncoder
                        pass
                    else:
                        encodedresult = "".join(cencoder.encode(c) for c in s)
                        cdecoder = codec_incrementaldecoder(encoding, "ignore")
                        decodedresult = u"".join(cdecoder.decode(c)
                                                 for c in encodedresult)
                        self.assertEqual(decodedresult, s,
                                         "encoding=%r" % encoding) 

# 需要导入模块: import codecs [as 别名]
# 或者: from codecs import iterencode [as 别名]
def test_incremental_encode(self):
        self.assertEqual(
            b"".join(codecs.iterencode("python.org", "idna")),
            b"python.org"
        )
        self.assertEqual(
            b"".join(codecs.iterencode("python.org.", "idna")),
            b"python.org."
        )
        self.assertEqual(
            b"".join(codecs.iterencode("pyth\xf6n.org.", "idna")),
            b"xn--pythn-mua.org."
        )
        self.assertEqual(
            b"".join(codecs.iterencode("pyth\xf6n.org.", "idna")),
            b"xn--pythn-mua.org."
        )

        encoder = codecs.getincrementalencoder("idna")()
        self.assertEqual(encoder.encode("\xe4x"), b"")
        self.assertEqual(encoder.encode("ample.org"), b"xn--xample-9ta.")
        self.assertEqual(encoder.encode("", True), b"org")

        encoder.reset()
        self.assertEqual(encoder.encode("\xe4x"), b"")
        self.assertEqual(encoder.encode("ample.org."), b"xn--xample-9ta.org.")
        self.assertEqual(encoder.encode("", True), b"")