Python cchardet.detect方法代码示例

# 需要导入模块: import cchardet [as 别名]
# 或者: from cchardet import detect [as 别名]
def get_encoding(self) -> str:
        ctype = self.headers.get(hdrs.CONTENT_TYPE, '').lower()
        mimetype = helpers.parse_mimetype(ctype)

        encoding = mimetype.parameters.get('charset')
        if encoding:
            try:
                codecs.lookup(encoding)
            except LookupError:
                encoding = None
        if not encoding:
            if mimetype.type == 'application' and mimetype.subtype == 'json':
                # RFC 7159 states that the default encoding is UTF-8.
                encoding = 'utf-8'
            else:
                encoding = chardet.detect(self._body)['encoding']
        if not encoding:
            encoding = 'utf-8'

        return encoding 

# 需要导入模块: import cchardet [as 别名]
# 或者: from cchardet import detect [as 别名]
def load_document_string(filename):
    '''load mock page from samples'''
    mypath = os.path.join(TEST_DIR, 'cache', filename)
    if not os.path.isfile(mypath):
        mypath = os.path.join(TEST_DIR, 'eval', filename)
    #if not os.path.isfile(mypath):
    #    mypath = os.path.join(TEST_DIR, 'additional', filename)
    try:
        with open(mypath, 'r') as inputf:
            htmlstring = inputf.read()
    # encoding/windows fix for the tests
    except UnicodeDecodeError:
        # read as binary
        with open(mypath, 'rb') as inputf:
            htmlbinary = inputf.read()
        guessed_encoding = chardet.detect(htmlbinary)['encoding']
        if guessed_encoding is not None:
            try:
                htmlstring = htmlbinary.decode(guessed_encoding)
            except UnicodeDecodeError:
                htmlstring = htmlbinary
        else:
            print('Encoding error')
    return htmlstring 

# 需要导入模块: import cchardet [as 别名]
# 或者: from cchardet import detect [as 别名]
def load_mock_page(url):
    '''Load mock page from samples'''
    try:
        with open(path.join(TEST_DIR, 'cache', MOCK_PAGES[url]), 'r') as inputf:
            htmlstring = inputf.read()
    # encoding/windows fix for the tests
    except UnicodeDecodeError:
        # read as binary
        with open(path.join(TEST_DIR, 'cache', MOCK_PAGES[url]), 'rb') as inputf:
            htmlbinary = inputf.read()
        guessed_encoding = chardet.detect(htmlbinary)['encoding']
        if guessed_encoding is not None:
            try:
                htmlstring = htmlbinary.decode(guessed_encoding)
            except UnicodeDecodeError:
                htmlstring = htmlbinary
        else:
            print('Encoding error')
    return htmlstring 

# 需要导入模块: import cchardet [as 别名]
# 或者: from cchardet import detect [as 别名]
def load_mock_page(url, xml_flag=False, langcheck=None, tei_output=False):
    '''load mock page from samples'''
    try:
        with open(os.path.join(TEST_DIR, 'cache', MOCK_PAGES[url]), 'r') as inputf:
            htmlstring = inputf.read()
    # encoding/windows fix for the tests
    except UnicodeDecodeError:
        # read as binary
        with open(os.path.join(TEST_DIR, 'cache', MOCK_PAGES[url]), 'rb') as inputf:
            htmlbinary = inputf.read()
        guessed_encoding = chardet.detect(htmlbinary)['encoding']
        if guessed_encoding is not None:
            try:
                htmlstring = htmlbinary.decode(guessed_encoding)
            except UnicodeDecodeError:
                htmlstring = htmlbinary
        else:
            print('Encoding error')
    result = extract(htmlstring, url,
                     record_id='0000',
                     no_fallback=False,
                     xml_output=xml_flag,
                     tei_output=tei_output,
                     target_language=langcheck)
    return result 

# 需要导入模块: import cchardet [as 别名]
# 或者: from cchardet import detect [as 别名]
def detect_encoding(sample, encoding=None):
    """Detect encoding of a byte string sample.
    """
    # To reduce tabulator import time
    try:
        from cchardet import detect
    except ImportError:
        from chardet import detect
    if encoding is not None:
        return normalize_encoding(sample, encoding)
    result = detect(sample)
    confidence = result['confidence'] or 0
    encoding = result['encoding'] or 'ascii'
    encoding = normalize_encoding(sample, encoding)
    if confidence < config.ENCODING_CONFIDENCE:
        encoding = config.DEFAULT_ENCODING
    if encoding == 'ascii':
        encoding = config.DEFAULT_ENCODING
    return encoding 

# 需要导入模块: import cchardet [as 别名]
# 或者: from cchardet import detect [as 别名]
def _get_html(cls, html, url, html_etree, params, **kwargs):
        if html:
            html = etree.HTML(html)
        elif url:
            if not kwargs.get('headers', None):
                kwargs['headers'] = {
                    "User-Agent": get_random_user_agent()
                }
            response = requests.get(url, params, **kwargs)
            response.raise_for_status()
            content = response.content
            charset = cchardet.detect(content)
            text = content.decode(charset['encoding'])
            html = etree.HTML(text)
        elif html_etree is not None:
            return html_etree
        else:
            raise ValueError("html(url or html_etree) is expected")
        return html 

# 需要导入模块: import cchardet [as 别名]
# 或者: from cchardet import detect [as 别名]
def decode_bytes(data: bytes, default_encoding: str = 'utf-8') -> str:
    """
    Decode the given bytes and return the decoded unicode string or raises
    `ActivityFailed`.

    When the chardet, or cchardet, packages are installed, we try to detect
    the encoding and use that instead of the default one (when the confidence
    is greater or equal than 50%).
    """
    encoding = default_encoding
    if HAS_CHARDET:
        detected = chardet.detect(data) or {}
        confidence = detected.get('confidence') or 0
        if confidence >= 0.5:
            encoding = detected['encoding']
            logger.debug(
                "Data encoding detected as '{}' "
                "with a confidence of {}".format(encoding, confidence))

    try:
        return data.decode(encoding)
    except UnicodeDecodeError:
        raise ActivityFailed(
            "Failed to decode bytes using encoding '{}'".format(encoding)) 

# 需要导入模块: import cchardet [as 别名]
# 或者: from cchardet import detect [as 别名]
def clone_url(url):
    """Get http code of url.

    :param url: url to clone
    :return:
    """
    # get html
    if '://' not in url:
        url = 'http://' + url

    r = requests.get(url)
    # We don't trust requests encoding so we use cchardet
    # to detect real encoding
    # Without it we got decode error (for example: baidu.com)
    r.encoding = cchardet.detect(r.content)['encoding']
    html = r.content.decode(r.encoding)

    # set relative url rule
    if '<base' not in html:
        html = html.replace('<head>', '<head><base href="%s" />' % url)

    return html 

# 需要导入模块: import cchardet [as 别名]
# 或者: from cchardet import detect [as 别名]
def fetch(self, url, params=None, guess_encoding=False, payload=None):
        self.content = None
        headers = {
            "User-Agent": r'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.116 Safari/537.36',
        }
        try:
            if payload:
                r = requests.post(url, headers=headers, params=params, data=payload, verify=False)
            else:
                r = requests.get(url, headers=headers, params=params, verify=False)
            if r.status_code == requests.codes.ok:
                self.content = r.content
                self.content_headers = r.headers
                if guess_encoding:
                    self.content_encoding = cchardet.detect(r.content).get("encoding")
                else:
                    self.content_encoding = r.encoding
            else:
                self.content = '?:' + str(r.status_code)
        except Exception:
            self.content = traceback.format_exc() 

# 需要导入模块: import cchardet [as 别名]
# 或者: from cchardet import detect [as 别名]
def my_get_charset(charset, body_bytes):
    detect = chardet.detect(body_bytes)
    if detect['encoding']:
        detect['encoding'] = detect['encoding'].lower()
    if detect['confidence']:
        detect['confidence'] = '{:.2f}'.format(detect['confidence'])

    for cset in (charset, detect['encoding'], 'utf-8'):
        if cset:
            try:
                codecs.lookup(cset)
                break
            except LookupError:
                pass
    else:
        cset = None

    return cset, detect 

# 需要导入模块: import cchardet [as 别名]
# 或者: from cchardet import detect [as 别名]
def my_decode(body_bytes, charset, detect):
    for cset in charset, detect['encoding']:
        if not cset:
            # encoding or detect may be None
            continue
        try:
            body = body_bytes.decode(encoding=cset)
            break
        except UnicodeDecodeError:
            # if we truncated the body, we could have caused the error:
            # UnicodeDecodeError: 'utf-8' codec can't decode byte 0xd9 in position 15: unexpected end of data
            # or encoding could be wrong, or the page could be defective
            pass
        except LookupError as e:
            if e.args[0].startswith('unknown encoding: '):  # e.g. viscii
                stats.stats_sum('content-encoding unknown: '+cset, 1)
                pass
    else:
        body = body_bytes.decode(encoding='utf-8', errors='replace')
        cset = 'utf-8 replace'
    return body, cset 

# 需要导入模块: import cchardet [as 别名]
# 或者: from cchardet import detect [as 别名]
def read_text_file(input_path):
    '''
    A function to read text, using cchardet to identify the encoding.
    '''
    with open(input_path, 'rb') as input_file:
        doc = input_file.read()
        # Try decoding as utf-8 first.  Then, if that doesn't work, try using
        # the cchardet module to auto-detect the encoding.
        try:
            doc = doc.decode('utf-8')
        except UnicodeDecodeError:
            chardet_output = cchardet.detect(doc)
            encoding = chardet_output['encoding']
            encoding_confidence = chardet_output['confidence']
            logging.debug('decoding {} as {} with {} confidence'
                          .format(input_path, encoding, encoding_confidence))
            doc = doc.decode(encoding)

    return doc 

# 需要导入模块: import cchardet [as 别名]
# 或者: from cchardet import detect [as 别名]
def encoding_detect(byte_content):
    """
    试图解析并返回二进制串的编码, 如果失败, 则返回 None
    :param byte_content: 待解码的二进制串
    :type byte_content: bytes
    :return: 编码类型或None
    :rtype: Union[str, None]
    """

    if force_decode_remote_using_encode is not None:
        return force_decode_remote_using_encode
    if possible_charsets:
        for charset in possible_charsets:
            try:
                byte_content.decode(encoding=charset)
            except:
                pass
            else:
                return charset
    if cchardet_available:  # detect the encoding using cchardet (if we have)
        return c_chardet(byte_content)['encoding']

    return None 

# 需要导入模块: import cchardet [as 别名]
# 或者: from cchardet import detect [as 别名]
def chardet_dammit(s):
        return cchardet.detect(s)['encoding'] 

# 需要导入模块: import cchardet [as 别名]
# 或者: from cchardet import detect [as 别名]
def detect_encoding(main, file_path):
    text = b''
    success = True

    with open(file_path, 'rb') as f:
        if main.settings_custom['auto_detection']['detection_settings']['number_lines_no_limit']:
            for line in f:
                text += line
        else:
            for i, line in enumerate(f):
                if i < main.settings_custom['auto_detection']['detection_settings']['number_lines']:
                    text += line
                else:
                    break

        encoding = cchardet.detect(text)['encoding']
        
        if encoding == 'SHIFT_JIS':
            # CP932
            encoding = chardet.detect(text)['encoding']

            if encoding != 'CP932':
                encoding = 'SHIFT_JIS'
        if encoding == 'EUC-TW':
            encoding = 'BIG5'
        elif encoding == 'ISO-2022-CN':
            encoding = 'GB18030'
        elif encoding == None:
            encoding = main.settings_custom['auto_detection']['default_settings']['default_encoding']

            success = False
        
    try:
        open(file_path, 'r', encoding = encoding)
    except:
        success = False

    encoding = encoding.lower()
    encoding = encoding.replace('-', '_')

    return encoding, success