本文整理匯總了Python中sacremoses.MosesDetokenizer方法的典型用法代碼示例。如果您正苦於以下問題：Python sacremoses.MosesDetokenizer方法的具體用法？Python sacremoses.MosesDetokenizer怎麽用？Python sacremoses.MosesDetokenizer使用的例子？那麽, 這裏精選的方法代碼示例或許可以為您提供幫助。您也可以進一步了解該方法所在類sacremoses的用法示例。
在下文中一共展示了sacremoses.MosesDetokenizer方法的6個代碼示例，這些例子默認根據受歡迎程度排序。您可以為喜歡或者感覺有用的代碼點讚，您的評價將有助於係統推薦出更棒的Python代碼示例。
示例1: __init__
# 需要導入模塊: import sacremoses [as 別名]
# 或者: from sacremoses import MosesDetokenizer [as 別名]
def __init__(self, *args, **kwargs):
if 'tokenize' in kwargs:
raise TypeError('``MosesEncoder`` does not take keyword argument ``tokenize``.')
if 'detokenize' in kwargs:
raise TypeError('``MosesEncoder`` does not take keyword argument ``detokenize``.')
try:
from sacremoses import MosesTokenizer
from sacremoses import MosesDetokenizer
except ImportError:
print("Please install SacreMoses. "
"See the docs at https://github.com/alvations/sacremoses for more information.")
raise
super().__init__(
*args,
tokenize=MosesTokenizer().tokenize,
detokenize=partial(MosesDetokenizer().detokenize, return_str=True),
**kwargs) 示例2: get_detokenizer
# 需要導入模塊: import sacremoses [as 別名]
# 或者: from sacremoses import MosesDetokenizer [as 別名]
def get_detokenizer():
from sacremoses import MosesDetokenizer
detok = MosesDetokenizer(lang='en')
return detok 示例3: main
# 需要導入模塊: import sacremoses [as 別名]
# 或者: from sacremoses import MosesDetokenizer [as 別名]
def main():
parser = argparse.ArgumentParser(description='')
parser.add_argument('files', nargs='*', help='input files')
args = parser.parse_args()
detok = sacremoses.MosesDetokenizer()
for line in fileinput.input(args.files, openhook=fileinput.hook_compressed):
print(detok.detokenize(line.strip().split(' ')).replace(' @', '').replace('@ ', '').replace(' =', '=').replace('= ', '=').replace(' – ', '–')) 示例4: __init__
# 需要導入模塊: import sacremoses [as 別名]
# 或者: from sacremoses import MosesDetokenizer [as 別名]
def __init__(self, args):
self.args = args
if getattr(args, 'moses_source_lang', None) is None:
args.moses_source_lang = getattr(args, 'source_lang', 'en')
if getattr(args, 'moses_target_lang', None) is None:
args.moses_target_lang = getattr(args, 'target_lang', 'en')
try:
from sacremoses import MosesTokenizer, MosesDetokenizer
self.tok = MosesTokenizer(args.moses_source_lang)
self.detok = MosesDetokenizer(args.moses_target_lang)
except ImportError:
raise ImportError('Please install Moses tokenizer with: pip install sacremoses') 示例5: __init__
# 需要導入模塊: import sacremoses [as 別名]
# 或者: from sacremoses import MosesDetokenizer [as 別名]
def __init__(self, escape: bool = False, *args, **kwargs):
self.escape = escape
self.tokenizer = MosesTokenizer()
self.detokenizer = MosesDetokenizer() 示例6: enable_moses
# 需要導入模塊: import sacremoses [as 別名]
# 或者: from sacremoses import MosesDetokenizer [as 別名]
def enable_moses(self, lang='en', tokenize=True, detokenize=True):
if tokenize:
self._moses_tok = MosesTokenizer(lang=lang)
else:
self._moses_tok = None
if detokenize:
self._moses_detok = MosesDetokenizer(lang=lang)
else:
self._moses_detok = None