Python langdetect.detect方法代码示例

# 需要导入模块: import langdetect [as 别名]
# 或者: from langdetect import detect [as 别名]
def google_translate(link):
    translator = Translator()	 # Google Translate API 
    pystring = " ".join(link[1:-2])
    lang = detect(pystring)
    if link[-1] == "english":
        id = "en"
    elif link[-1] == "spanish":
        id = "es"
    elif link[-1] == "french":
        id = "fr"
    elif link[-1] == "german":
        id = "de"
    elif link[-1] == "italian":
        id = "it"
    elif link[-1] == "portugese" or link[-1] == "portuguese":
        id = "pt"
    else:
        id = "en"
    translated = translator.translate(pystring, src=lang, dest=id)	# To Translate the given language to the required language
    print(translated.text)	# Print the translated script
    try:
        speak.say("The translated text is "+translated.text)
        speak.runAndWait()
    except:
        print("Error speaking, here is the translated text: {}".format(translated.text)) 

# 需要导入模块: import langdetect [as 别名]
# 或者: from langdetect import detect [as 别名]
def tag_reviews_language(self):

        print('%s: tag reviews language' % time.strftime("%Y/%m/%d-%H:%M:%S"))

        if os.path.exists(Constants.LANGUAGE_RECORDS_FILE):
            print('Records have already been tagged with language field')
            self.records = \
                ETLUtils.load_json_file(Constants.LANGUAGE_RECORDS_FILE)
            return

        DetectorFactory.seed = 0

        for record in self.records:
            try:
                language = langdetect.detect(record[Constants.TEXT_FIELD])
            except LangDetectException:
                language = 'unknown'
            record[Constants.LANGUAGE_FIELD] = language

        ETLUtils.save_json_file(Constants.LANGUAGE_RECORDS_FILE, self.records) 

# 需要导入模块: import langdetect [as 别名]
# 或者: from langdetect import detect [as 别名]
def translate(message, cmd, option, text):
    """
    指定した文字列を翻訳する

    http://docs.microsofttranslator.com/text-translate.html#!/default/get_Translate
    """
    if text in ('help', 'list'):
        return

    lang = 'ja'
    if option:
        # 指定した言語に翻訳する
        _, lang = option.split('-', 1)
    elif detect(text) in ('ja', 'ko'):
        # 漢字が多いと日本語なのに ko と判定される
        # 日本語の場合は英語に翻訳する
        lang = 'en'

    url = API_BASE_URL + 'Translate'
    headers = {
        'Ocp-Apim-Subscription-Key': settings.TRANSLATOR_API_KEY,
    }
    query = {
        'to': lang,
        'text': text,
    }
    r = requests.get(url, headers=headers, params=query)

    if r.status_code == 400:
        # エラーが発生したので内容を表示する
        error_message = r.text
        if "Message: 'to' must be a valid language" in error_message:
            botsend(message, '`{}` は無効な言語です'.format(lang))
        else:
            botsend(message, 'エラーが発生しました\n```\n{}\n```'.format(r.text))
        return

    tree = ET.fromstring(r.text)
    botsend(message, tree.text) 

# 需要导入模块: import langdetect [as 别名]
# 或者: from langdetect import detect [as 别名]
def language(text, override_language=None):
    if override_language:
        return override_language

    try:
        return detect(text)
    except LangDetectException:
        return 'en' 

# 需要导入模块: import langdetect [as 别名]
# 或者: from langdetect import detect [as 别名]
def detectLanguageAndEncode(s):
        # mod = importlib.import_module("langdetect")
        # detect_method = getattr(mod, "detect")
        # code = detect_method(s)
        from langdetect import detect
        code = detect(s)
        code_index = TextLangDetectTransformer.language_codes.index(
            code) if code in TextLangDetectTransformer.language_codes else -1
        return code_index 

# 需要导入模块: import langdetect [as 别名]
# 或者: from langdetect import detect [as 别名]
def main(args): 

    with open(args.input[0], "r", errors='ignore') as frs:
        with open(args.input[1], "r", errors='ignore') as frt:
            data1 = [line for line in frs]
            data2 = [line for line in frt]

    if len(data1) != len(data2):
        print(len(data1), len(data2))
        raise ValueError("length of two files are not equal")

    src_lang = args.lang[0]
    trg_lang = args.lang[1]

    with open(args.input[0] + "." + args.suffix, "w") as fws:
        with open(args.input[1] + "." + args.suffix, "w") as fwt:
            for i in range(len(data1)):
                if i % 10000 == 0:
                    print(i, end = '\r')
                try:
                    line1 = data1[i].replace("@@ ", "")
                    line2 = data2[i].replace("@@ ", "")
                    length1 = len(line1.split(' '))
                    length2 = len(line2.split(' '))
                    if length1 / length2 < 0.7 or length2 / length1 < 0.7:
                        continue
                    if detect(line1) != src_lang or detect(line2) != trg_lang:
                        continue
                    fws.write(data1[i])
                    fwt.write(data2[i])
                except:
                    continue 

# 需要导入模块: import langdetect [as 别名]
# 或者: from langdetect import detect [as 别名]
def main():
    parent_dir = TRAIN_DIR / 'cmn'

    for target_file in ['tatoeba.txt', 'w2c.txt']:

        hant_file = open(TRAIN_DIR / 'cmn-hant' / target_file, mode='w')
        hans_file = open(TRAIN_DIR / 'cmn-hans' / target_file, mode='w')

        with open(parent_dir / target_file) as f:
            for line in f:
                text = line.rstrip()
                if not text:
                    continue

                try:
                    lang = langdetect.detect(text)
                except langdetect.lang_detect_exception.LangDetectException:
                    continue

                if lang in {'zh-tw', 'ko'}:
                    text = HanziConv.toTraditional(text)
                    hant_file.write(text)
                    hant_file.write('\n')
                elif lang == 'zh-cn':
                    text = HanziConv.toSimplified(text)
                    hans_file.write(text)
                    hans_file.write('\n')

        hant_file.close()
        hans_file.close() 

# 需要导入模块: import langdetect [as 别名]
# 或者: from langdetect import detect [as 别名]
def detect_lang(x):
    try:
        language = langdetect.detect(x)
    except:
        language = 'unk'
    return language


# language detection with textblob package 

# 需要导入模块: import langdetect [as 别名]
# 或者: from langdetect import detect [as 别名]
def get_response(self, query):
        try:
            probable_language = detect(query)
        except LangDetectException:
            probable_language = ""
        logging.info("Probable language is {}".format(probable_language))
        return self.get_response_from_chatbot(query, probable_language) 

# 需要导入模块: import langdetect [as 别名]
# 或者: from langdetect import detect [as 别名]
def detect_language(text):
    from langdetect import DetectorFactory, detect
    # Stay consistent between builds
    DetectorFactory.seed = 0

    return detect(text) 

# 需要导入模块: import langdetect [as 别名]
# 或者: from langdetect import detect [as 别名]
def _safe_detect(text):
    try:
        lang = detect(text)
    except Exception:
        lang = 'en'
    return lang 

# 需要导入模块: import langdetect [as 别名]
# 或者: from langdetect import detect [as 别名]
def detect_language(self):
    """Langdetect is non-deterministic, so to achieve a higher probability
    we attempt detection multiple times and only report success if we get identical results.
    """
    if self.language:
        return

    try:
        nondet_attempts = [detect(self.fulltext) for i in range(0,2)]
        is_unique = len(set(nondet_attempts)) == 1
        self.language = nondet_attempts[0] if is_unique else False
    except:
        pass 

# 需要导入模块: import langdetect [as 别名]
# 或者: from langdetect import detect [as 别名]
def detect_language(texts):
        texts = pd.Series(texts)
        language_codes = texts.apply(langdetect.detect).value_counts()
        return language_codes.index[0] 

# 需要导入模块: import langdetect [as 别名]
# 或者: from langdetect import detect [as 别名]
def _remove_stopwords(self, text):
        if text == '':
            return text

        if self.language_code:
            language_code = self.language_code

        elif self.language == 'multi':
            language_code = langdetect.detect(text)

        sw = self.get_stopwords(language_code)

        return ' '.join(word for word in text.split() if word not in sw) 

# 需要导入模块: import langdetect [as 别名]
# 或者: from langdetect import detect [as 别名]
def separate_scripts(text, scripts):
    """
    Takes in a string and an iterable of language tags and returns an array of string paragraphs
    separated by language. The first language in scripts is the default. The paragraphs may be either
    html (separated by <p> or triple <br><br><br> tags) or text (separated by \n or dash).

    :param text: The plain text or html to separate paragraphs in by language.
    :param scripts: Iterable of allowed languages.
    :return:
    """
    # separate the text by paragraphs, matching to select html and plain text delimiters in data
    paragraphs = re.split(r'(</p><p>|\n|</p>|<p>| – |<br><br><br>)+', text)
    separated = {script: '' for script in scripts}
    # the first language given is the default one
    last_language = scripts[0]
    last_paragraph = ''
    for paragraph in paragraphs:
        if paragraph in (r'</p><p>', r'</p>' r'\n', r'<p>', r'<br><br><br>'):
            # skip paragraph breaks to prevent misdetection
            separated[last_language] += paragraph
            last_paragraph = paragraph
            continue
        # replace any misleading tags left
        paragraph_stripped = re.sub(r'(<(/)?strong>)|(<br>)+|&amp;|<a href=.*">|</a>', ' ', paragraph)
        try:
            language = detect(paragraph_stripped)
        except LangDetectException:
            # an exception means no language could be detected
            language = last_language
        # langdetect maps "Simplified Chinese" to "zh-cn"
        # However, we store it as "zh_hans"
        if language == "zh-cn":
            language = "zh_hans"
        if language not in scripts:
            # only detect allowed languages, no exceptions
            language = last_language
        if language != last_language:
            # fix html paragraph breaks after language change
            logger.debug('supported language detected: ' + language)
            if last_paragraph in (r'</p><p>', r'</p>', r'<p>'):
                separated[last_language] = re.sub(r'<p>$', '', separated[last_language])
                separated[language] += r'<p>'
            # remove useless dashes after language change
            if last_paragraph in (r' – ',):
                separated[last_language] = re.sub(r' – $', '', separated[last_language])
            # replace the awful triple-<br>
            if last_paragraph in (r'<br><br><br>',):
                separated[last_language] = re.sub(r'<br><br><br>$', '', separated[last_language])
                separated[last_language] += r'</p>'
                separated[language] += r'<p>'
        separated[language] += paragraph
        last_language = language
        last_paragraph = paragraph
    return separated