當前位置: 首頁>>代碼示例>>Python>>正文

Python ptbtokenizer.PTBTokenizer方法代碼示例

本文整理匯總了Python中tokenizer.ptbtokenizer.PTBTokenizer方法的典型用法代碼示例。如果您正苦於以下問題：Python ptbtokenizer.PTBTokenizer方法的具體用法？Python ptbtokenizer.PTBTokenizer怎麽用？Python ptbtokenizer.PTBTokenizer使用的例子？那麽, 這裏精選的方法代碼示例或許可以為您提供幫助。您也可以進一步了解該方法所在類tokenizer.ptbtokenizer的用法示例。


在下文中一共展示了ptbtokenizer.PTBTokenizer方法的10個代碼示例，這些例子默認根據受歡迎程度排序。您可以為喜歡或者感覺有用的代碼點讚，您的評價將有助於係統推薦出更棒的Python代碼示例。

示例1: __init__

# 需要導入模塊: from tokenizer import ptbtokenizer [as 別名]
# 或者: from tokenizer.ptbtokenizer import PTBTokenizer [as 別名]
def __init__(self, gts, res, df):
        print 'tokenization...'
        tokenizer = PTBTokenizer('gts')
        _gts = tokenizer.tokenize(gts)
        print 'tokenized refs'
        tokenizer = PTBTokenizer('res')
        _res = tokenizer.tokenize(res)
        print 'tokenized cands'

        self.gts = _gts
        self.res = _res
        self.df = df
開發者ID:xiadingZ，項目名稱:video-caption-openNMT.pytorch，代碼行數:14，代碼來源:eval.py

示例2: evaluate

# 需要導入模塊: from tokenizer import ptbtokenizer [as 別名]
# 或者: from tokenizer.ptbtokenizer import PTBTokenizer [as 別名]
def evaluate(self):

        # imgIds = self.coco.getImgIds()
        gts = dict(zip(range(0, len(self.predicted_list)), self.predicted_list))
        res = dict(zip(range(0, len(self.label_list)), self.label_list))

        # =================================================
        # Set up scorers
        # =================================================
        print 'tokenization...'
        tokenizer = PTBTokenizer()
        gts = tokenizer.tokenize(gts)
        res = tokenizer.tokenize(res)

        # =================================================
        # Set up scorers
        # =================================================
        print 'setting up scorers...'
        scorers = [
            (Bleu(4), ["Bleu_1", "Bleu_2", "Bleu_3", "Bleu_4"]),
            (Meteor(),"METEOR"),
            (Rouge(), "ROUGE_L"),
            (Cider(), "CIDEr")
        ]

        # =================================================
        # Compute scores
        # =================================================
        for scorer, method in scorers:
            print 'computing %s score...'%(scorer.method())
            score, scores = scorer.compute_score(gts, res)
            if type(method) == list:
                for sc, scs, m in zip(score, scores, method):
                    self.setEval(sc, m)
                    self.set_textid_to_eval(scs, gts.keys(), m)
                    print "%s: %0.3f"%(m, sc)
            else:
                self.setEval(score, method)
                self.set_textid_to_eval(scores, gts.keys(), method)
                print "%s: %0.3f"%(method, score)
        self.set_eval()
開發者ID:hadyelsahar，項目名稱:Zeroshot-QuestionGeneration，代碼行數:43，代碼來源:eval.py

示例3: eval

# 需要導入模塊: from tokenizer import ptbtokenizer [as 別名]
# 或者: from tokenizer.ptbtokenizer import PTBTokenizer [as 別名]
def eval(target, refers, scorer, tokenizer = PTBTokenizer(), use_private = False):
    """docstring for main"""
    k = 'single'

    res_single = {k:[target]}
    gts = {k:refers}

    # =================================================
    # Convert to COCO format
    # =================================================
    gts = to_coco(gts, res_single.keys())
    res = to_coco(res_single, res_single.keys())

    # =================================================
    # Set up scorers
    # =================================================
    #print 'tokenization...'
    #tokenizer = PTBTokenizer()
    #import ipdb; ipdb.set_trace()
    gts  = tokenizer.tokenize(gts)
    res = tokenizer.tokenize(res)

    if use_private:
        # initialize the meteor.jar
        score, scores = scorer._compute_score(gts, res)
    else:
        score, scores = scorer.compute_score(gts, res)
    print score
    return score, scores
開發者ID:raingo，項目名稱:TGIF-Release，代碼行數:31，代碼來源:eval.py

示例4: __init__

# 需要導入模塊: from tokenizer import ptbtokenizer [as 別名]
# 或者: from tokenizer.ptbtokenizer import PTBTokenizer [as 別名]
def __init__(self, gts, res, df):
        print('tokenization...')
        tokenizer = PTBTokenizer('gts')
        _gts = tokenizer.tokenize(gts)
        print('tokenized refs')
        tokenizer = PTBTokenizer('res')
        _res = tokenizer.tokenize(res)
        print('tokenized cands')

        self.gts = _gts
        self.res = _res
        self.df = df
開發者ID:jiasenlu，項目名稱:NeuralBabyTalk，代碼行數:14，代碼來源:eval.py

示例5: evaluate

# 需要導入模塊: from tokenizer import ptbtokenizer [as 別名]
# 或者: from tokenizer.ptbtokenizer import PTBTokenizer [as 別名]
def evaluate(self):
        imgIds = self.params['image_id']
        # imgIds = self.coco.getImgIds()
        gts = {}
        res = {}
        for imgId in imgIds:
            gts[imgId] = self.coco.imgToAnns[imgId]
            res[imgId] = self.cocoRes.imgToAnns[imgId]

        # =================================================
        # Set up scorers
        # =================================================
        print 'tokenization...'
        tokenizer = PTBTokenizer()
        gts  = tokenizer.tokenize(gts)
        res = tokenizer.tokenize(res)

        # =================================================
        # Set up scorers
        # =================================================
        print 'setting up scorers...'
        scorers = [
            (Bleu(4), ["Bleu_1", "Bleu_2", "Bleu_3", "Bleu_4"]),
            (Meteor(),"METEOR"),
            (Rouge(), "ROUGE_L"),
            (Cider(), "CIDEr")
        ]

        # =================================================
        # Compute scores
        # =================================================
        eval = {}
        for scorer, method in scorers:
            print 'computing %s score...'%(scorer.method())
            score, scores = scorer.compute_score(gts, res)
            if type(method) == list:
                for sc, scs, m in zip(score, scores, method):
                    self.setEval(sc, m)
                    self.setImgToEvalImgs(scs, imgIds, m)
                    print "%s: %0.3f"%(m, sc)
            else:
                self.setEval(score, method)
                self.setImgToEvalImgs(scores, imgIds, method)
                print "%s: %0.3f"%(method, score)
        self.setEvalImgs()
開發者ID:InnerPeace-Wu，項目名稱:densecap-tensorflow，代碼行數:47，代碼來源:eval.py

示例6: main

# 需要導入模塊: from tokenizer import ptbtokenizer [as 別名]
# 或者: from tokenizer.ptbtokenizer import PTBTokenizer [as 別名]
def main():

    import sys
    res_path = sys.argv[1]

    gt_path = osp.join(this_dir, 'tgif-v1.0.tsv')
    test_list_path = osp.join(this_dir, 'splits', 'test.txt')

    test_keys = load_list(test_list_path)
    all_sents = load_sentences(gt_path)
    res = load_sentences(res_path)

    # make sure res has and only has single sentence
    # for all testing keys
    gts = {}
    for key in test_keys:
        gts[key] = all_sents[key]
        if key in res:
            res[key] = [res[key][0]]
        else:
            res[key] = [""]

    # =================================================
    # Convert to COCO format
    # =================================================
    gts = to_coco(gts, res.keys())
    res = to_coco(res, res.keys())

    # =================================================
    # Set up scorers
    # =================================================
    print 'tokenization...'
    tokenizer = PTBTokenizer()
    gts  = tokenizer.tokenize(gts)
    res = tokenizer.tokenize(res)

    # =================================================
    # Set up scorers
    # =================================================
    print 'setting up scorers...'
    scorers = [
        (Bleu(4), ["Bleu_1", "Bleu_2", "Bleu_3", "Bleu_4"]),
        (Meteor(),"METEOR"),
        (Rouge(), "ROUGE_L"),
        (Cider(), "CIDEr")
    ]

    # =================================================
    # Compute scores
    # =================================================
    eval = {}
    for scorer, method in scorers:
        print 'computing %s score...'%(scorer.method())
        score, scores = scorer.compute_score(gts, res)
        if type(method) == list:
            for sc, scs, m in zip(score, scores, method):
                print "%s: %0.3f"%(m, sc)
        else:
            print "%s: %0.3f"%(method, score)
開發者ID:raingo，項目名稱:TGIF-Release，代碼行數:61，代碼來源:eval.py

示例7: evaluate

# 需要導入模塊: from tokenizer import ptbtokenizer [as 別名]
# 或者: from tokenizer.ptbtokenizer import PTBTokenizer [as 別名]
def evaluate(self):
        imgIds = self.params['image_id']
        # imgIds = self.coco.getImgIds()
        gts = {}
        res = {}
        for imgId in imgIds:
            gts[imgId] = self.coco.imgToAnns[imgId]
            res[imgId] = self.cocoRes.imgToAnns[imgId]

        # =================================================
        # Set up scorers
        # =================================================
        print('tokenization...')
        tokenizer = PTBTokenizer()
        gts  = tokenizer.tokenize(gts)
        res = tokenizer.tokenize(res)

        # =================================================
        # Set up scorers
        # =================================================
        print('setting up scorers...')
        scorers = [
            (Bleu(4), ["Bleu_1", "Bleu_2", "Bleu_3", "Bleu_4"]),
            (Meteor(),"METEOR"),
            (Rouge(), "ROUGE_L"),
            (Cider(), "CIDEr")
        ]

        # =================================================
        # Compute scores
        # =================================================
        for scorer, method in scorers:
            print('computing %s score...'%(scorer.method()))
            score, scores = scorer.compute_score(gts, res)
            if type(method) == list:
                for sc, scs, m in zip(score, scores, method):
                    self.setEval(sc, m)
                    self.setImgToEvalImgs(scs, gts.keys(), m)
                    print("%s: %0.3f"%(m, sc))
            else:
                self.setEval(score, method)
                self.setImgToEvalImgs(scores, gts.keys(), method)
                print("%s: %0.3f"%(method, score))
        self.setEvalImgs()
開發者ID:microsoft，項目名稱:DialoGPT，代碼行數:46，代碼來源:eval.py

示例8: evaluate

# 需要導入模塊: from tokenizer import ptbtokenizer [as 別名]
# 或者: from tokenizer.ptbtokenizer import PTBTokenizer [as 別名]
def evaluate(self):
        imgIds = self.params['image_id']
        # imgIds = self.coco.getImgIds()
        gts = {}
        res = {}
        for imgId in imgIds:
            gts[imgId] = self.coco.imgToAnns[imgId]
            res[imgId] = self.cocoRes.imgToAnns[imgId]

        # =================================================
        # Set up scorers
        # =================================================
        print 'tokenization...'
        tokenizer = PTBTokenizer()
        gts = tokenizer.tokenize(gts)
        res = tokenizer.tokenize(res)

        # =================================================
        # Set up scorers
        # =================================================
        print 'setting up scorers...'
        scorers = [
            (Bleu(4), ["Bleu_1", "Bleu_2", "Bleu_3", "Bleu_4"]),
            (Meteor(), "METEOR"),
            (Rouge(), "ROUGE_L"),
            (Cider(), "CIDEr")
        ]

        # =================================================
        # Compute scores
        # =================================================
        for scorer, method in scorers:
            print 'computing %s score...' % (scorer.method())
            score, scores = scorer.compute_score(gts, res)
            if type(method) == list:
                for sc, scs, m in zip(score, scores, method):
                    self.setEval(sc, m)
                    self.setImgToEvalImgs(scs, gts.keys(), m)
                    print "%s: %0.3f" % (m, sc)
            else:
                self.setEval(score, method)
                self.setImgToEvalImgs(scores, gts.keys(), method)
                print "%s: %0.3f" % (method, score)
        self.setEvalImgs()
開發者ID:sheffieldnlp，項目名稱:deepQuest，代碼行數:46，代碼來源:eval.py

示例9: evaluate

# 需要導入模塊: from tokenizer import ptbtokenizer [as 別名]
# 或者: from tokenizer.ptbtokenizer import PTBTokenizer [as 別名]
def evaluate(self):
        imgIds = self.params[self.imgidStr]
        # imgIds = self.coco.getImgIds()
        gts = {}
        res = {}
        for imgId in imgIds:
            gts[imgId] = self.coco.imgToAnns[imgId]
            res[imgId] = self.cocoRes.imgToAnns[imgId]

        # =================================================
        # Set up scorers
        # =================================================
        print 'tokenization...'
        tokenizer = PTBTokenizer()
        gts  = tokenizer.tokenize(gts)
        res = tokenizer.tokenize(res)

        # =================================================
        # Set up scorers
        # =================================================
        print 'setting up scorers...'
        scorers = [
            (Bleu(4), ["Bleu_1", "Bleu_2", "Bleu_3", "Bleu_4"]),
            (Meteor(),"METEOR"),
            (Rouge(), "ROUGE_L"),
            (Cider(), "CIDEr"),
            (Spice(), "SPICE")
        ]

        # =================================================
        # Compute scores
        # =================================================
        eval = {}
        for scorer, method in scorers:
            print 'computing %s score...'%(scorer.method())
            score, scores = scorer.compute_score(gts, res)
            if type(method) == list:
                for sc, scs, m in zip(score, scores, method):
                    self.setEval(sc, m)
                    self.setImgToEvalImgs(scs, gts.keys(), m)
                    print "%s: %0.3f"%(m, sc)
            else:
                self.setEval(score, method)
                self.setImgToEvalImgs(scores, gts.keys(), method)
                print "%s: %0.3f"%(method, score)
        self.setEvalImgs()
開發者ID:rakshithShetty，項目名稱:captionGAN，代碼行數:48，代碼來源:eval.py

示例10: evaluate

# 需要導入模塊: from tokenizer import ptbtokenizer [as 別名]
# 或者: from tokenizer.ptbtokenizer import PTBTokenizer [as 別名]
def evaluate(self):
        imgIds = self.params['image_id']
        # imgIds = self.coco.getImgIds()
        gts = {}
        res = {}
        for imgId in imgIds:
            gts[imgId] = self.coco.imgToAnns[imgId]
            res[imgId] = self.cocoRes.imgToAnns[imgId]

        # =================================================
        # Set up scorers
        # =================================================
        print 'tokenization...'
        tokenizer = PTBTokenizer()
        gts  = tokenizer.tokenize(gts)
        res = tokenizer.tokenize(res)

        # =================================================
        # Set up scorers
        # =================================================
        print 'setting up scorers...'
        scorers = [
            (Bleu(4), ["Bleu_1", "Bleu_2", "Bleu_3", "Bleu_4"]),
            (Meteor(),"METEOR"),
            (Rouge(), "ROUGE_L"),
            (Cider(), "CIDEr")
        ]

        # =================================================
        # Compute scores
        # =================================================
        for scorer, method in scorers:
            print 'computing %s score...'%(scorer.method())
            score, scores = scorer.compute_score(gts, res)
            if type(method) == list:
                for sc, scs, m in zip(score, scores, method):
                    self.setEval(sc, m)
                    self.setImgToEvalImgs(scs, gts.keys(), m)
                    print "%s: %0.3f"%(m, sc)
            else:
                self.setEval(score, method)
                self.setImgToEvalImgs(scores, gts.keys(), method)
                print "%s: %0.3f"%(method, score)
        self.setEvalImgs()
開發者ID:yicheng-w，項目名稱:CommonSenseMultiHopQA，代碼行數:46，代碼來源:eval.py


注：本文中的tokenizer.ptbtokenizer.PTBTokenizer方法示例由純淨天空整理自Github/MSDocs等開源代碼及文檔管理平台，相關代碼片段篩選自各路編程大神貢獻的開源項目，源碼版權歸原作者所有，傳播和使用請參考對應項目的License；未經允許，請勿轉載。