Python opts.py方法代码示例

# 需要导入模块: import opts [as 别名]
# 或者: from opts import py [as 别名]
def evaluate(best_val_checkpoint_path):
    # python translate.py -src data/multi30k/test2016.en.atok -output pred.txt \
    #                     -replace_unk -tgt=data/multi30k/test2016.de.atok -report_bleu -gpu 2
    #                     -model saves/2018-02-09-enc:Rev-dec:Rev-et:RevGRU-dt:RevGRU-h:300-el:1-dl:1-em:300-atn:general-cxt:slice_emb-sl:20-ef1:0.875-ef2:0.875-df1:0.875-df2:0.875/best_checkpoint.pt

    base_dir = os.path.dirname(best_val_checkpoint_path)

    if '600' in best_val_checkpoint_path:
        test_output = subprocess.run(['python', 'translate.py', '-src', 'data/en-de/IWSLT16.TED.tst2014.en-de.en.tok.low',
                                      '-output', os.path.join(base_dir, 'test_pred.txt'), '-replace_unk', '-tgt', 'data/en-de/IWSLT16.TED.tst2014.en-de.de.tok.low',
                                      '-report_bleu', '-gpu', str(opt.gpuid[0]), '-model', best_val_checkpoint_path], stdout=subprocess.PIPE)

        test_output_string = test_output.stdout.decode('utf-8')
        print(test_output_string)

        # Also save the whole stdout string for reference
        with open(os.path.join(base_dir, 'test_stdout.txt'), 'w') as f:
            f.write('{}\n'.format(test_output_string))

        val_output = subprocess.run(['python', 'translate.py', '-src', 'data/en-de/IWSLT16.TED.tst2013.en-de.en.tok.low',
                                     '-output', os.path.join(base_dir, 'val_pred.txt'), '-replace_unk', '-tgt', 'data/en-de/IWSLT16.TED.tst2013.en-de.de.tok.low',
                                     '-report_bleu', '-gpu', str(opt.gpuid[0]), '-model', best_val_checkpoint_path], stdout=subprocess.PIPE)

        val_output_string = val_output.stdout.decode('utf-8')
        print(val_output_string)
    else:
        test_output = subprocess.run(['python', 'translate.py', '-src', 'data/multi30k/test2016.en.tok.low',
                                      '-output', os.path.join(base_dir, 'test_pred.txt'), '-replace_unk', '-tgt', 'data/multi30k/test2016.de.tok.low',
                                      '-report_bleu', '-gpu', str(opt.gpuid[0]), '-model', best_val_checkpoint_path], stdout=subprocess.PIPE)

        test_output_string = test_output.stdout.decode('utf-8')
        print(test_output_string)

        # Also save the whole stdout string for reference
        with open(os.path.join(base_dir, 'test_stdout.txt'), 'w') as f:
            f.write('{}\n'.format(test_output_string))

        val_output = subprocess.run(['python', 'translate.py', '-src', 'data/multi30k/val.en.tok.low',
                                     '-output', os.path.join(base_dir, 'val_pred.txt'), '-replace_unk', '-tgt', 'data/multi30k/val.de.tok.low',
                                     '-report_bleu', '-gpu', str(opt.gpuid[0]), '-model', best_val_checkpoint_path], stdout=subprocess.PIPE)

        val_output_string = val_output.stdout.decode('utf-8')
        print(val_output_string)

    # Also save the whole stdout string for reference
    with open(os.path.join(base_dir, 'val_stdout.txt'), 'w') as f:
        f.write('{}\n'.format(val_output_string))

    val_bleu = extract_bleu_score(val_output_string)
    test_bleu = extract_bleu_score(test_output_string)

    with open(os.path.join(base_dir, 'result.txt'), 'w') as f:
        f.write('{} {}\n'.format(val_bleu, test_bleu))

    print('Val BLEU: {} | Test BLEU: {}'.format(val_bleu, test_bleu)) 

# 需要导入模块: import opts [as 别名]
# 或者: from opts import py [as 别名]
def main():

    # Load train and validate data.
    train_dataset = load_dataset("train")
    valid_dataset = load_dataset("valid")
    print(' * maximum batch size: %d' % opt.batch_size)

    # Load checkpoint if we resume from a previous training.
    if opt.train_from:
        print('Loading checkpoint from %s' % opt.train_from)
        checkpoint = torch.load(opt.train_from,
                                map_location=lambda storage, loc: storage)
        model_opt = checkpoint['opt']
        # I don't like reassigning attributes of opt: it's not clear.
        opt.start_epoch = checkpoint['epoch'] + 1
    else:
        checkpoint = None
        model_opt = opt

    # Load fields generated from preprocess phase.
    fields = load_fields(train_dataset, valid_dataset, checkpoint)

    # Report src/tgt features.
    collect_report_features(fields)

    # Build model.
    model = build_model(model_opt, opt, fields, checkpoint)
    tally_parameters(model)
    check_save_model_path()

    # Build optimizer.
    optim = build_optim(model, checkpoint)

    # load embeddings
    # NOTE you need to comment/uncomment the following section to use word embeddings!!!!!!
    # NOTE DO NOT USE THOSE WORD EMBED OPTIONS IN opts.py because they do not work!!!!!!
    fields['src'].vocab.load_vectors(wv_type='glove.42B', wv_dim=300)
    fields['tgt'].vocab.load_vectors(wv_type='glove.42B', wv_dim=300)
    model.encoder.embeddings.word_lut.weight.data.copy_(fields['src'].vocab.vectors.cuda())
    model.decoder.embeddings.word_lut.weight.data.copy_(fields['tgt'].vocab.vectors.cuda())
    model.encoder.embeddings.word_lut.weight.requires_grad = False
    model.decoder.embeddings.word_lut.weight.requires_grad = False

    # Do training.
    train_model(model, train_dataset, valid_dataset, fields, optim, model_opt)