本文整理汇总了Python中sacremoses.MosesDetokenizer方法的典型用法代码示例。如果您正苦于以下问题：Python sacremoses.MosesDetokenizer方法的具体用法？Python sacremoses.MosesDetokenizer怎么用？Python sacremoses.MosesDetokenizer使用的例子？那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类sacremoses的用法示例。
在下文中一共展示了sacremoses.MosesDetokenizer方法的6个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: __init__
# 需要导入模块: import sacremoses [as 别名]
# 或者: from sacremoses import MosesDetokenizer [as 别名]
def __init__(self, *args, **kwargs):
if 'tokenize' in kwargs:
raise TypeError('``MosesEncoder`` does not take keyword argument ``tokenize``.')
if 'detokenize' in kwargs:
raise TypeError('``MosesEncoder`` does not take keyword argument ``detokenize``.')
try:
from sacremoses import MosesTokenizer
from sacremoses import MosesDetokenizer
except ImportError:
print("Please install SacreMoses. "
"See the docs at https://github.com/alvations/sacremoses for more information.")
raise
super().__init__(
*args,
tokenize=MosesTokenizer().tokenize,
detokenize=partial(MosesDetokenizer().detokenize, return_str=True),
**kwargs) 示例2: get_detokenizer
# 需要导入模块: import sacremoses [as 别名]
# 或者: from sacremoses import MosesDetokenizer [as 别名]
def get_detokenizer():
from sacremoses import MosesDetokenizer
detok = MosesDetokenizer(lang='en')
return detok 示例3: main
# 需要导入模块: import sacremoses [as 别名]
# 或者: from sacremoses import MosesDetokenizer [as 别名]
def main():
parser = argparse.ArgumentParser(description='')
parser.add_argument('files', nargs='*', help='input files')
args = parser.parse_args()
detok = sacremoses.MosesDetokenizer()
for line in fileinput.input(args.files, openhook=fileinput.hook_compressed):
print(detok.detokenize(line.strip().split(' ')).replace(' @', '').replace('@ ', '').replace(' =', '=').replace('= ', '=').replace(' – ', '–')) 示例4: __init__
# 需要导入模块: import sacremoses [as 别名]
# 或者: from sacremoses import MosesDetokenizer [as 别名]
def __init__(self, args):
self.args = args
if getattr(args, 'moses_source_lang', None) is None:
args.moses_source_lang = getattr(args, 'source_lang', 'en')
if getattr(args, 'moses_target_lang', None) is None:
args.moses_target_lang = getattr(args, 'target_lang', 'en')
try:
from sacremoses import MosesTokenizer, MosesDetokenizer
self.tok = MosesTokenizer(args.moses_source_lang)
self.detok = MosesDetokenizer(args.moses_target_lang)
except ImportError:
raise ImportError('Please install Moses tokenizer with: pip install sacremoses') 示例5: __init__
# 需要导入模块: import sacremoses [as 别名]
# 或者: from sacremoses import MosesDetokenizer [as 别名]
def __init__(self, escape: bool = False, *args, **kwargs):
self.escape = escape
self.tokenizer = MosesTokenizer()
self.detokenizer = MosesDetokenizer() 示例6: enable_moses
# 需要导入模块: import sacremoses [as 别名]
# 或者: from sacremoses import MosesDetokenizer [as 别名]
def enable_moses(self, lang='en', tokenize=True, detokenize=True):
if tokenize:
self._moses_tok = MosesTokenizer(lang=lang)
else:
self._moses_tok = None
if detokenize:
self._moses_detok = MosesDetokenizer(lang=lang)
else:
self._moses_detok = None