Python pyrouge.Rouge155方法代码示例

# 需要导入模块: import pyrouge [as 别名]
# 或者: from pyrouge import Rouge155 [as 别名]
def _rouge(system_dir, gold_dir):
    # Run rouge
    r = Rouge155()
    r.system_dir = system_dir
    r.model_dir = gold_dir
    r.system_filename_pattern = '([a-zA-Z0-9]*).model'
    r.model_filename_pattern = '#ID#.gold'
    output = r.convert_and_evaluate(rouge_args="-e /address/to/rouge/data/directory/rouge/data -a -c 95 -m -n 4 -w 1.2")
    # print output
    output_dict = r.output_to_dict(output)
    # print output_dict
    
    # avg_rscore = 0
    # if FLAGS.rouge_reward_fscore:
    #     avg_rscore = (output_dict["rouge_1_f_score"]+output_dict["rouge_2_f_score"]+
    #                   output_dict["rouge_3_f_score"]+output_dict["rouge_4_f_score"]+
    #                   output_dict["rouge_l_f_score"])/5.0
    # else:
    #     avg_rscore = (output_dict["rouge_1_recall"]+output_dict["rouge_2_recall"]+
    #                   output_dict["rouge_3_recall"]+output_dict["rouge_4_recall"]+
    #                   output_dict["rouge_l_recall"])/5.0

    avg_rscore = (output_dict["rouge_1_f_score"]+output_dict["rouge_2_f_score"]+output_dict["rouge_l_f_score"])/3.0

    return avg_rscore 

# 需要导入模块: import pyrouge [as 别名]
# 或者: from pyrouge import Rouge155 [as 别名]
def evaluate_summ(article):
    ref=''
    for k in wt(data['summaries'][article])[:20]:
        ref=ref+' '+k
    gen_sum = generateText(summarize(train_data["article"][article]))
    print("-----------------------------------------------------")
    print("Original summary")
    print(ref)
    print("-----------------------------------------------------")
    print("Generated summary")
    print(gen_sum)
    print("-----------------------------------------------------")
    rouge = Rouge155()
    score = rouge.score_summary(ref, gen_sum)
    print("Rouge1 Score: ",score)
        
#######################################################################################
################################ Train model and test##################################
####################################################################################### 

# 需要导入模块: import pyrouge [as 别名]
# 或者: from pyrouge import Rouge155 [as 别名]
def _rouge(system_dir, gold_dir):
    # Run rouge
    r = Rouge155()
    r.system_dir = system_dir
    r.model_dir = gold_dir
    r.system_filename_pattern = '([a-zA-Z0-9]*).model'
    r.model_filename_pattern = '#ID#.gold'
    output = r.convert_and_evaluate(rouge_args="-e Code/neuralsum/ROUGE_evaluation/rouge/data -a -c 95 -m -n 4 -w 1.2")
    # print output
    output_dict = r.output_to_dict(output)
    # print output_dict
    
    avg_rscore = 0
    
    avg_rscore = (output_dict["rouge_1_recall"]+output_dict["rouge_2_recall"]+
                  output_dict["rouge_3_recall"]+output_dict["rouge_4_recall"]+
                  output_dict["rouge_l_recall"])/5.0
    return avg_rscore 

# 需要导入模块: import pyrouge [as 别名]
# 或者: from pyrouge import Rouge155 [as 别名]
def test_wrong_model_pattern(self):
        rouge = Rouge155()
        rouge.system_dir = add_data_path("systems")
        rouge.model_dir = add_data_path("models_plain")
        rouge.system_filename_pattern = "SL.P.10.R.11.SL062003-(\d+).html"
        rouge.model_filename_pattern = "SL.P.10.R.[A-D].SL062003-#ID#.html"
        with self.assertRaises(Exception) as context:
            rouge.evaluate()
        match_string = (
            r"Could not find any model summaries for the system "
            r"summary with ID " + "(\d+)" + r". Specified model filename "
            r"pattern was: " + re.escape(rouge.model_filename_pattern))
        try:
            assert_regex = self.assertRegex
        except AttributeError:
            assert_regex = self.assertRegexpMatches
        assert_regex(str(context.exception), re.compile(match_string)) 

# 需要导入模块: import pyrouge [as 别名]
# 或者: from pyrouge import Rouge155 [as 别名]
def test_rouge_for_plain_text(self):
        model_dir = add_data_path("models_plain")
        system_dir = add_data_path("systems_plain")
        pyrouge_command = (
            "pyrouge_evaluate_plain_text_files -m {} -s {} -sfp "
            "D(\d+).M.100.T.A -mfp D#ID#.M.100.T.[A-Z] -id 1".format(
                model_dir, system_dir))
        pyrouge_output = check_output_clean(pyrouge_command.split())
        rouge = Rouge155()
        config_file = add_data_path("config_test2.xml")
        rouge_command = (
            "{bin} -e {data} -c 95 -2 -1 -U -r 1000 -n 4 -w 1.2 "
            "-a -m {xml}".format(
                bin=rouge.bin_path,
                data=rouge.data_dir,
                xml=config_file))
        orig_rouge_output = check_output_clean(rouge_command.split())
        self.assertEqual(pyrouge_output, orig_rouge_output) 

# 需要导入模块: import pyrouge [as 别名]
# 或者: from pyrouge import Rouge155 [as 别名]
def test_options(self):
        rouge = Rouge155()
        model_dir = add_data_path("models_plain")
        system_dir = add_data_path("systems_plain")
        config_file = add_data_path("config_test2.xml")
        command_part1 = (
            "pyrouge_evaluate_plain_text_files -m {} -s {} -sfp "
            "D(\d+).M.100.T.A -mfp D#ID#.M.100.T.[A-Z] -id 1 -rargs".format(
                model_dir, system_dir))

        command_part2 = [
            "\"-e {data} -c 90 -2 -1 -U -r 1000 -n 2 -w 1.2 "
            "-a -m {xml}\"".format(
                data=rouge.data_dir, xml=config_file)]

        pyrouge_command = command_part1.split() + command_part2
        pyrouge_output = check_output_clean(pyrouge_command)
        rouge_command = (
            "{bin} -e {data} -c 90 -2 -1 -U -r 1000 -n 2 -w 1.2 "
            "-a -m {xml}".format(
                bin=rouge.bin_path, data=rouge.data_dir, xml=config_file))
        orig_rouge_output = check_output_clean(rouge_command.split())
        self.assertEqual(pyrouge_output, orig_rouge_output) 

# 需要导入模块: import pyrouge [as 别名]
# 或者: from pyrouge import Rouge155 [as 别名]
def main(_):
  rouge = pyrouge.Rouge155()
  rouge.log.setLevel(logging.ERROR)
  rouge.system_filename_pattern = "rouge.(\\d+).txt"
  rouge.model_filename_pattern = "rouge.[A-Z].#ID#.txt"

  tf.logging.set_verbosity(tf.logging.INFO)

  tmpdir = mkdtemp()
  tf.logging.info("tmpdir: %s" % tmpdir)
  # system = decodes/predictions
  system_dir = os.path.join(tmpdir, "system")
  # model = targets/gold
  model_dir = os.path.join(tmpdir, "model")
  os.mkdir(system_dir)
  os.mkdir(model_dir)

  rouge.system_dir = system_dir
  rouge.model_dir = model_dir

  prep_data(rouge.system_dir, rouge.model_dir)

  rouge_scores = rouge.convert_and_evaluate()
  rouge_scores = rouge.output_to_dict(rouge_scores)
  for prefix in ["rouge_1", "rouge_2", "rouge_l"]:
    for suffix in ["f_score", "precision", "recall"]:
      key = "_".join([prefix, suffix])
      tf.logging.info("%s: %.4f" % (key, rouge_scores[key]))

  # clean up after pyrouge
  shutil.rmtree(tmpdir)
  shutil.rmtree(rouge._config_dir)  # pylint: disable=protected-access
  shutil.rmtree(os.path.split(rouge._system_dir)[0])  # pylint: disable=protected-access 

# 需要导入模块: import pyrouge [as 别名]
# 或者: from pyrouge import Rouge155 [as 别名]
def rouge_eval(ref_dir, dec_dir):
  """Evaluate the files in ref_dir and dec_dir with pyrouge, returning results_dict"""
  r = pyrouge.Rouge155()
  r.model_filename_pattern = '#ID#_reference.txt'
  r.system_filename_pattern = '(\d+)_decoded.txt'
  r.model_dir = ref_dir
  r.system_dir = dec_dir
  logging.getLogger('global').setLevel(logging.WARNING) # silence pyrouge logging
  rouge_results = r.convert_and_evaluate()
  return r.output_to_dict(rouge_results) 

# 需要导入模块: import pyrouge [as 别名]
# 或者: from pyrouge import Rouge155 [as 别名]
def test_rouge(cand, ref):
    """Calculate ROUGE scores of sequences passed as an iterator
       e.g. a list of str, an open file, StringIO or even sys.stdin
    """
    current_time = time.strftime('%Y-%m-%d-%H-%M-%S', time.localtime())
    tmp_dir = ".rouge-tmp-{}".format(current_time)
    try:
        if not os.path.isdir(tmp_dir):
            os.mkdir(tmp_dir)
            os.mkdir(tmp_dir + "/candidate")
            os.mkdir(tmp_dir + "/reference")
        candidates = [line.strip() for line in cand]
        references = [line.strip() for line in ref]
        assert len(candidates) == len(references)
        cnt = len(candidates)
        for i in range(cnt):
            if len(references[i]) < 1:
                continue
            with open(tmp_dir + "/candidate/cand.{}.txt".format(i), "w",
                      encoding="utf-8") as f:
                f.write(candidates[i])
            with open(tmp_dir + "/reference/ref.{}.txt".format(i), "w",
                      encoding="utf-8") as f:
                f.write(references[i])
        r = pyrouge.Rouge155()
        r.model_dir = tmp_dir + "/reference/"
        r.system_dir = tmp_dir + "/candidate/"
        r.model_filename_pattern = 'ref.#ID#.txt'
        r.system_filename_pattern = 'cand.(\d+).txt'
        rouge_results = r.convert_and_evaluate()
        results_dict = r.output_to_dict(rouge_results)
        return results_dict
    finally:
        pass
        if os.path.isdir(tmp_dir):
            shutil.rmtree(tmp_dir) 

# 需要导入模块: import pyrouge [as 别名]
# 或者: from pyrouge import Rouge155 [as 别名]
def test_rouge(cand, ref):
    """Calculate ROUGE scores of sequences passed as an iterator
       e.g. a list of str, an open file, StringIO or even sys.stdin
    """
    current_time = time.strftime('%Y-%m-%d-%H-%M-%S', time.localtime())
    tmp_dir = ".rouge-tmp-{}".format(current_time)
    try:
        if not os.path.isdir(tmp_dir):
            os.mkdir(tmp_dir)
            os.mkdir(tmp_dir + "/candidate")
            os.mkdir(tmp_dir + "/reference")
        candidates = [line.strip() for line in cand]
        references = [line.strip() for line in ref]
        assert len(candidates) == len(references)
        cnt = len(candidates)
        for i in range(cnt):
            if len(references[i]) < 1:
                continue
            with open(tmp_dir + "/candidate/cand.{}.txt".format(i), "w",
                      encoding="utf-8") as f:
                f.write(candidates[i])
            with open(tmp_dir + "/reference/ref.{}.txt".format(i), "w",
                      encoding="utf-8") as f:
                f.write(references[i])
        r = pyrouge.Rouge155()
        r.model_dir = tmp_dir + "/reference/"
        r.system_dir = tmp_dir + "/candidate/"
        r.model_filename_pattern = 'ref.#ID#.txt'
        r.system_filename_pattern = r'cand.(\d+).txt'
        rouge_results = r.convert_and_evaluate()
        results_dict = r.output_to_dict(rouge_results)
        return results_dict
    finally:
        pass
        if os.path.isdir(tmp_dir):
            shutil.rmtree(tmp_dir) 

# 需要导入模块: import pyrouge [as 别名]
# 或者: from pyrouge import Rouge155 [as 别名]
def print_pyrouge(config):
    logging.getLogger('global').setLevel(logging.WARNING) # silence pyrouge logging

    r = pyrouge.Rouge155()
    r.system_filename_pattern = '(\d+).txt'
    r.model_filename_pattern = '#ID#.txt'
    
    r.system_dir = config.outdir
    r.model_dir = config.refdir

    rouge_results = r.convert_and_evaluate()
    rouge_dict = r.output_to_dict(rouge_results)
    
    print(rouge_results) 

# 需要导入模块: import pyrouge [as 别名]
# 或者: from pyrouge import Rouge155 [as 别名]
def test_rouge(cand_file, ref_file):
    f_cand = open(cand_file, encoding="utf-8")
    f_ref = open(ref_file, encoding="utf-8")
    current_time = time.strftime('%Y-%m-%d-%H-%M-%S', time.localtime())
    tmp_dir = ".rouge-tmp-{}".format(current_time)
    try:
        if not os.path.isdir(tmp_dir):
            os.mkdir(tmp_dir)
            os.mkdir(tmp_dir + "/candidate")
            os.mkdir(tmp_dir + "/reference")
        candidates = [line.strip() for line in f_cand]
        references = [line.strip() for line in f_ref]
        assert len(candidates) == len(references)
        cnt = len(candidates)
        for i in range(cnt):
            if len(references[i]) < 1:
                continue
            with open(tmp_dir + "/candidate/cand.{}.txt".format(i), "w",
                      encoding="utf-8") as f:
                f.write(candidates[i])
            with open(tmp_dir + "/reference/ref.{}.txt".format(i), "w",
                      encoding="utf-8") as f:
                f.write(references[i])
        f_cand.close()
        f_ref.close()
        r = pyrouge.Rouge155()
        r.model_dir = tmp_dir + "/reference/"
        r.system_dir = tmp_dir + "/candidate/"
        r.model_filename_pattern = 'ref.#ID#.txt'
        r.system_filename_pattern = 'cand.(\d+).txt'
        rouge_results = r.convert_and_evaluate()
        results_dict = r.output_to_dict(rouge_results)
        return results_dict
    finally:
        pass
        if os.path.isdir(tmp_dir):
            shutil.rmtree(tmp_dir) 

# 需要导入模块: import pyrouge [as 别名]
# 或者: from pyrouge import Rouge155 [as 别名]
def rouge_eval(ref_dir, dec_dir):
    """Evaluate the files in ref_dir and dec_dir with pyrouge, returning results_dict"""
    r = pyrouge.Rouge155()
    r.model_filename_pattern = '#ID#_reference.txt'
    r.system_filename_pattern = '(\d+)_decoded.txt'
    r.model_dir = ref_dir
    r.system_dir = dec_dir
    logging.getLogger('global').setLevel(
        logging.WARNING)  # silence pyrouge logging
    rouge_results = r.convert_and_evaluate()
    return r.output_to_dict(rouge_results) 

# 需要导入模块: import pyrouge [as 别名]
# 或者: from pyrouge import Rouge155 [as 别名]
def rouge_eval(ref_dir, dec_dir):
  """Evaluate the files in ref_dir and dec_dir with pyrouge, returning results_dict"""
  r = pyrouge.Rouge155()
  r.model_filename_pattern = '#ID#_reference.txt'
  r.system_filename_pattern = '([\W\w]+)_decoded.txt'
  r.model_dir = ref_dir
  r.system_dir = dec_dir
  logging.getLogger('global').setLevel(logging.WARNING) # silence pyrouge logging
  rouge_results = r.convert_and_evaluate()
  return r.output_to_dict(rouge_results) 

# 需要导入模块: import pyrouge [as 别名]
# 或者: from pyrouge import Rouge155 [as 别名]
def rouge(reference, candidate, log_path, print_log, config):
    assert len(reference) == len(candidate)

    ref_dir = log_path + 'reference/'
    cand_dir = log_path + 'candidate/'
    if not os.path.exists(ref_dir):
        os.mkdir(ref_dir)
    if not os.path.exists(cand_dir):
        os.mkdir(cand_dir)

    for i in range(len(reference)):
        with codecs.open(ref_dir+"%06d_reference.txt" % i, 'w', 'utf-8') as f:
            f.write(" ".join(reference[i]).replace(' <\s> ', '\n') + '\n')
        with codecs.open(cand_dir+"%06d_candidate.txt" % i, 'w', 'utf-8') as f:
            f.write(" ".join(candidate[i]).replace(' <\s> ', '\n').replace('<unk>', 'UNK') + '\n')

    r = pyrouge.Rouge155()
    r.model_filename_pattern = '#ID#_reference.txt'
    r.system_filename_pattern = '(\d+)_candidate.txt'
    r.model_dir = ref_dir
    r.system_dir = cand_dir
    logging.getLogger('global').setLevel(logging.WARNING)
    rouge_results = r.convert_and_evaluate()
    scores = r.output_to_dict(rouge_results)
    recall = [round(scores["rouge_1_recall"] * 100, 2),
              round(scores["rouge_2_recall"] * 100, 2),
              round(scores["rouge_l_recall"] * 100, 2)]
    precision = [round(scores["rouge_1_precision"] * 100, 2),
                 round(scores["rouge_2_precision"] * 100, 2),
                 round(scores["rouge_l_precision"] * 100, 2)]
    f_score = [round(scores["rouge_1_f_score"] * 100, 2),
               round(scores["rouge_2_f_score"] * 100, 2),
               round(scores["rouge_l_f_score"] * 100, 2)]
    print_log("F_measure: %s Recall: %s Precision: %s\n"
              % (str(f_score), str(recall), str(precision)))

    return f_score[:], recall[:], precision[:]