Python codecs.getreader方法代码示例

# 需要导入模块: import codecs [as 别名]
# 或者: from codecs import getreader [as 别名]
def _get_external_data(url):
    result = {}
    try:
        # urlopen might fail if it runs into redirections,
        # because of Python issue #13696. Fixed in locators
        # using a custom redirect handler.
        resp = urlopen(url)
        headers = resp.info()
        ct = headers.get('Content-Type')
        if not ct.startswith('application/json'):
            logger.debug('Unexpected response for JSON request: %s', ct)
        else:
            reader = codecs.getreader('utf-8')(resp)
            #data = reader.read().decode('utf-8')
            #result = json.loads(data)
            result = json.load(reader)
    except Exception as e:
        logger.exception('Failed to get external data for %s: %s', url, e)
    return result 

# 需要导入模块: import codecs [as 别名]
# 或者: from codecs import getreader [as 别名]
def metadata(self):
        pathname = os.path.join(self.dirname, self.filename)
        name_ver = '%s-%s' % (self.name, self.version)
        info_dir = '%s.dist-info' % name_ver
        wrapper = codecs.getreader('utf-8')
        with ZipFile(pathname, 'r') as zf:
            wheel_metadata = self.get_wheel_metadata(zf)
            wv = wheel_metadata['Wheel-Version'].split('.', 1)
            file_version = tuple([int(i) for i in wv])
            if file_version < (1, 1):
                fn = 'METADATA'
            else:
                fn = METADATA_FILENAME
            try:
                metadata_filename = posixpath.join(info_dir, fn)
                with zf.open(metadata_filename) as bf:
                    wf = wrapper(bf)
                    result = Metadata(fileobj=wf)
            except KeyError:
                raise ValueError('Invalid wheel, because %s is '
                                 'missing' % fn)
        return result 

# 需要导入模块: import codecs [as 别名]
# 或者: from codecs import getreader [as 别名]
def _get_external_data(url):
    result = {}
    try:
        # urlopen might fail if it runs into redirections,
        # because of Python issue #13696. Fixed in locators
        # using a custom redirect handler.
        resp = urlopen(url)
        headers = resp.info()
        if headers.get('Content-Type') != 'application/json':
            logger.debug('Unexpected response for JSON request')
        else:
            reader = codecs.getreader('utf-8')(resp)
            #data = reader.read().decode('utf-8')
            #result = json.loads(data)
            result = json.load(reader)
    except Exception as e:
        logger.exception('Failed to get external data for %s: %s', url, e)
    return result 

# 需要导入模块: import codecs [as 别名]
# 或者: from codecs import getreader [as 别名]
def in_out(args,multiple_files=False):
    """Open the input/output data streams. If multiple_files is set to
    True, returns an iterator over lines. If set to False, returns an open file.
    This distinction is needed because validator.py checks the newlines property and
    needs to get the input as a file, but the other scripts just need the lines
    so they can work with several files.
    """
    #Decide where to get the data from
    if args.input is None or args.input=="-": #Stdin
        inp=codecs.getreader("utf-8")(os.fdopen(0,"U")) #Switched universal newlines on
    else: #File name given
        if multiple_files:
            inp_raw=fileinput.input(files=args.input,mode="U")
            inp=(line.decode("utf-8") for line in inp_raw)
        else:
            inp_raw=open(args.input,mode="U")
            inp=codecs.getreader("utf-8")(inp_raw)
    #inp is now an iterator over lines, giving unicode strings

    if args.output is None or args.output=="-": #stdout
        out=codecs.getwriter("utf-8")(sys.stdout)
    else: #File name given
        out=codecs.open(args.output,"w","utf-8")
    return inp,out 

# 需要导入模块: import codecs [as 别名]
# 或者: from codecs import getreader [as 别名]
def test_incrementalencoder(self):
        UTF8Reader = codecs.getreader('utf-8')
        for sizehint in [None] + range(1, 33) + \
                        [64, 128, 256, 512, 1024]:
            istream = UTF8Reader(StringIO(self.tstring[1]))
            ostream = StringIO()
            encoder = self.incrementalencoder()
            while 1:
                if sizehint is not None:
                    data = istream.read(sizehint)
                else:
                    data = istream.read()

                if not data:
                    break
                e = encoder.encode(data)
                ostream.write(e)

            self.assertEqual(ostream.getvalue(), self.tstring[0]) 

# 需要导入模块: import codecs [as 别名]
# 或者: from codecs import getreader [as 别名]
def test_streamwriter(self):
        readfuncs = ('read', 'readline', 'readlines')
        UTF8Reader = codecs.getreader('utf-8')
        for name in readfuncs:
            for sizehint in [None] + range(1, 33) + \
                            [64, 128, 256, 512, 1024]:
                istream = UTF8Reader(StringIO(self.tstring[1]))
                ostream = self.writer(StringIO())
                func = getattr(istream, name)
                while 1:
                    if sizehint is not None:
                        data = func(sizehint)
                    else:
                        data = func()

                    if not data:
                        break
                    if name == "readlines":
                        ostream.writelines(data)
                    else:
                        ostream.write(data)

                self.assertEqual(ostream.getvalue(), self.tstring[0]) 

# 需要导入模块: import codecs [as 别名]
# 或者: from codecs import getreader [as 别名]
def test_bug1098990_b(self):
        s1 = u"aaaaaaaaaaaaaaaaaaaaaaaa\r\n"
        s2 = u"bbbbbbbbbbbbbbbbbbbbbbbb\r\n"
        s3 = u"stillokay:bbbbxx\r\n"
        s4 = u"broken!!!!badbad\r\n"
        s5 = u"againokay.\r\n"

        s = (s1+s2+s3+s4+s5).encode(self.encoding)
        stream = StringIO.StringIO(s)
        reader = codecs.getreader(self.encoding)(stream)
        self.assertEqual(reader.readline(), s1)
        self.assertEqual(reader.readline(), s2)
        self.assertEqual(reader.readline(), s3)
        self.assertEqual(reader.readline(), s4)
        self.assertEqual(reader.readline(), s5)
        self.assertEqual(reader.readline(), u"") 

# 需要导入模块: import codecs [as 别名]
# 或者: from codecs import getreader [as 别名]
def test_stream_bom(self):
        unistring = u"ABC\u00A1\u2200XYZ"
        bytestring = codecs.BOM_UTF8 + "ABC\xC2\xA1\xE2\x88\x80XYZ"

        reader = codecs.getreader("utf-8-sig")
        for sizehint in [None] + range(1, 11) + \
                        [64, 128, 256, 512, 1024]:
            istream = reader(StringIO.StringIO(bytestring))
            ostream = StringIO.StringIO()
            while 1:
                if sizehint is not None:
                    data = istream.read(sizehint)
                else:
                    data = istream.read()

                if not data:
                    break
                ostream.write(data)

            got = ostream.getvalue()
            self.assertEqual(got, unistring) 

# 需要导入模块: import codecs [as 别名]
# 或者: from codecs import getreader [as 别名]
def test_all(self):
        api = (
            "encode", "decode",
            "register", "CodecInfo", "Codec", "IncrementalEncoder",
            "IncrementalDecoder", "StreamReader", "StreamWriter", "lookup",
            "getencoder", "getdecoder", "getincrementalencoder",
            "getincrementaldecoder", "getreader", "getwriter",
            "register_error", "lookup_error",
            "strict_errors", "replace_errors", "ignore_errors",
            "xmlcharrefreplace_errors", "backslashreplace_errors",
            "open", "EncodedFile",
            "iterencode", "iterdecode",
            "BOM", "BOM_BE", "BOM_LE",
            "BOM_UTF8", "BOM_UTF16", "BOM_UTF16_BE", "BOM_UTF16_LE",
            "BOM_UTF32", "BOM_UTF32_BE", "BOM_UTF32_LE",
            "BOM32_BE", "BOM32_LE", "BOM64_BE", "BOM64_LE",  # Undocumented
            "StreamReaderWriter", "StreamRecoder",
        )
        self.assertEqual(sorted(api), sorted(codecs.__all__))
        for api in codecs.__all__:
            getattr(codecs, api) 

# 需要导入模块: import codecs [as 别名]
# 或者: from codecs import getreader [as 别名]
def __iter__(self):
        if self.filename:
            file = codecs.open(self.filename, 'r', 'utf-8', errors='ignore')
        else:
            file = codecs.getreader('utf-8')(sys.stdin)
        sent = []
        for line in file:
            line = line.strip()
            if line:
                sent.append(line.split('\t'))
            else:
                yield sent
                sent = []
        if sent:                # just in case
            yield sent
        if self.filename:
            file.close() 

# 需要导入模块: import codecs [as 别名]
# 或者: from codecs import getreader [as 别名]
def main():
    """Main entry function."""
    if len(sys.argv) < 3:
        print('Usage: <project-name> <filetype> <list-of-path to traverse>')
        print('\tfiletype can be python/cpp/all')
        exit(-1)
    _HELPER.project_name = sys.argv[1]
    file_type = sys.argv[2]
    allow_type = []
    if file_type == 'python' or file_type == 'all':
        allow_type += [x for x in PYTHON_SUFFIX]
    if file_type == 'cpp' or file_type == 'all':
        allow_type += [x for x in CXX_SUFFIX]
    allow_type = set(allow_type)
    if os.name != 'nt':
        sys.stderr = codecs.StreamReaderWriter(sys.stderr,
                                               codecs.getreader('utf8'),
                                               codecs.getwriter('utf8'),
                                               'replace')
    for path in sys.argv[3:]:
        if os.path.isfile(path):
            process(path, allow_type)
        else:
            for root, dirs, files in os.walk(path):
                for name in files:
                    process(os.path.join(root, name), allow_type)

    nerr = _HELPER.print_summary(sys.stderr)
    sys.exit(nerr > 0) 

# 需要导入模块: import codecs [as 别名]
# 或者: from codecs import getreader [as 别名]
def reformat_namelist(filename, out=None):
    if out is None:
        out = sys.stdout
    if filename == '-':
        _reformat_namelist(codecs.getreader('utf8')(sys.stdin), out)
        return
    with codecs.open(filename, 'r', encoding='utf-8') as f:
        _reformat_namelist(f, out) 

# 需要导入模块: import codecs [as 别名]
# 或者: from codecs import getreader [as 别名]
def decoding_stream(stream, encoding, errors='strict'):
    try:
        reader_cls = codecs.getreader(encoding or sys.getdefaultencoding())
    except LookupError:
        reader_cls = codecs.getreader(sys.getdefaultencoding())
    return reader_cls(stream, errors) 

# 需要导入模块: import codecs [as 别名]
# 或者: from codecs import getreader [as 别名]
def __init__(self, f, encoding):
        self.reader = codecs.getreader(encoding)(f) 

# 需要导入模块: import codecs [as 别名]
# 或者: from codecs import getreader [as 别名]
def __init__(self, **kwargs):
        if 'stream' in kwargs:
            stream = kwargs['stream']
            if sys.version_info[0] >= 3:
                # needs to be a text stream
                stream = codecs.getreader('utf-8')(stream)
            self.stream = stream
        else:
            self.stream = _csv_open(kwargs['path'], 'r')
        self.reader = csv.reader(self.stream, **self.defaults)