Python Evaluator.eval方法代码示例

# 需要导入模块: from evaluator import Evaluator [as 别名]
# 或者: from evaluator.Evaluator import eval [as 别名]
class Perceptron(BasePerceptron):

    def __init__(self, decoder, train, dev, output, iter = 1 , avg = True):
        BasePerceptron.__init__(self, decoder, [train], output=output)
        self.iter = iter
        self.avg = avg
        self.eval = Evaluator(decoder, [dev])


    @staticmethod
    def cmdline_perc(decoder):
        return Perceptron(decoder, train = FLAGS.train, dev = FLAGS.dev,
                          output = FLAGS.out, iter = FLAGS.iter, avg = FLAGS.avg )
        
    def avg_weights(self):
        return self.weights - self.allweights * (1/self.c)
        
    def train(self):

        starttime = time.time()

        print >> logs, "starting perceptron at", time.ctime()

        best_prec = 0
        for it in xrange(1, self.iter+1):

            print >> logs, "iteration %d starts..............%s" % (it, time.ctime())

            iterstarttime = time.time()
            num_updates, early_updates = self.one_pass_on_train()

            print >> logs, "iteration %d training finished at %s. now evaluating on dev..." % (it, time.ctime())
            avgweights = self.avg_weights() if self.avg else self.weights
            if FLAGS.debuglevel >= 2:
                print >> logs, "avg w=", avgweights
            self.decoder.set_model_weights(avgweights)
            prec = self.eval.eval().compute_score()
            
            print >> logs, "at iteration {0}, updates= {1} (early {4}), dev= {2}, |w|= {3}, time= {5:.3f}h acctime= {6:.3f}h"\
                  .format(it, num_updates, prec, len(avgweights), early_updates, \
                          (time.time() - iterstarttime)/3600, (time.time() - starttime)/3600.)
            logs.flush()

            if prec > best_prec:
                best_prec = prec
                best_it = it
                best_wlen = len(avgweights)
                print >> logs, "new high at iteration {0}: {1}. Dumping Weights...".format(it, prec)
                self.dump(avgweights)

            self.decoder.set_model_weights(self.weights) # restore non-avg

        print >> logs, "peaked at iteration {0}: {1}, |bestw|= {2}.".format(best_it, best_prec, best_wlen)
        print >> logs, "perceptron training of %d iterations finished on %s (took %.2f hours)"  % \
              (it, time.ctime(), (time.time() - starttime)/3600.)

# 需要导入模块: from evaluator import Evaluator [as 别名]
# 或者: from evaluator.Evaluator import eval [as 别名]
def main():
    f = open('try_3.txt','w')
    g = open('accs.txt', 'w')
    g.close()
    task = MarioTask("testbed", initMarioMode = 2)
    task.env.initMarioMode = 2
    task.env.levelDifficulty = 1

    results = [] 
    names = [] 


    
    iterations = 5
    rounds = 2
    learning_samples = 2
    eval_samples = 3


    if args['noisy']:
        prefix = '-noisy-sup-eval'
        dire = './training_data_noisy/'
        agent = NoisySupervise(IT, useKMM = False)
    else:
        prefix = '-sup-eval'
        dire = './training_data/'
        agent = Supervise(IT,useKMM = False)

    if args['linear']:
        agent.learner.linear = True
        prefix = 'svc' + prefix
    else:
        agent.learner.linear = False
        prefix = 'dt' + prefix

    exp = EpisodicExperiment(task, agent) 
    E = Evaluator(agent,exp)
    sl_data, sup_data, acc, loss, js = E.eval(rounds = rounds, iterations = iterations, 
        learning_samples=learning_samples, eval_samples=eval_samples, prefix = prefix,
        directory = dire)

    np.save('./data/' + prefix + '-sl_data.npy', sl_data)
    np.save('./data/' + prefix + '-acc.npy', acc)
    np.save('./data/' + prefix + '-loss.npy', loss)
    np.save('./data/' + prefix + '-js.npy', js)


    analysis = Analysis()
    analysis.get_perf(sl_data, range(iterations))
    analysis.plot(names=['Supervised Learning'], label='Reward', filename='./results/' + prefix + '-return_plots.eps')#, ylims=[0, 1600])

    acc_a = Analysis()
    acc_a.get_perf(acc, range(iterations))
    acc_a.plot(names=['Supervised Learning Acc.'], label='Accuracy', filename='./results/' + prefix + '-acc_plots.eps', ylims=[0, 1])

    loss_a = Analysis()
    loss_a.get_perf(loss, range(iterations))
    loss_a.plot(names=['Supervised Learning loss'], label='Loss', filename='./results/' + prefix + '-loss_plots.eps', ylims=[0, 1])
    
    js_a = Analysis()
    js_a.get_perf(js, range(iterations))
    js_a.plot(names=['Supervised Learning'], label='J()', filename='./results/' + prefix + '-js_plots.eps')


       

    #agent.saveModel()
    print "finished"

# 需要导入模块: from evaluator import Evaluator [as 别名]
# 或者: from evaluator.Evaluator import eval [as 别名]
 def checkEvaluation(self, s, expected, precision=0.00, symbols = None):
     parser = Parser(OperatorFactory(), OperandFactory(), symbols)
     sut = Evaluator(parser)
     result = sut.eval(s)
     ok_(result, expected, precision)

# 需要导入模块: from evaluator import Evaluator [as 别名]
# 或者: from evaluator.Evaluator import eval [as 别名]
class BaseCRF(trainer.BaseTrainer):

    def __init__(self, decoder, train, dev, output, iter = 1):
        trainer.BaseTrainer.__init__(self, decoder, [train], output=output)
        self.iter = iter
        self.eval = Evaluator(decoder, [dev])
        self.u = 0
        #self.N = 10
        self.weight_norm = 0.0
        
    def set_oracle_files(self, oraclefiles):
      self.oraclefiles = oraclefiles

    def set_configuration(self, corpus_size, batch_size):
      self.N = corpus_size
      self.B = batch_size

    def get_eta(self, k):
      # exponential eta rate
      return eta_0 * (alpha ** (k/ self.N))

    @staticmethod
    def clip_weights(fv, range):
      """keep weights within a fixed update range"""
      for f in fv:
        fv[f] = min(fv[f],range) if fv[f] >0 else max(fv[f],-range)
      return fv


    def do_update(self, full_update, k):
      
      eta = self.get_eta(k)
      print >>logs, "Doing batch update %s, Eta %s" % (k, eta)
      if DEBUG:
        all_feat = []
        for feat in full_update:
          all_feat.append((abs(full_update[feat]), full_update[feat], feat))
        all_feat.sort(reverse=True)
        self.dump(dict([(f,v2) for (v,v2,f) in all_feat[0:10]]))

      #update = self.clip_weights(full_update, 1.0)
      update = full_update
      self.weights += eta * update
      if USE_L1:
        self.u += (regularization_constant/self.N) * eta
        for feat in update:
          prev = self.weights[feat]
          if self.weights[feat] > 0 :
            self.weights[feat] = max(0, self.weights[feat] - (self.u + self.q[feat]))
          else:
            self.weights[feat] = min(0, self.weights[feat] + (self.u - self.q[feat]))
          
          self.q[feat] += self.weights[feat] - prev
          

          # keep \sum |w| up to date
          assert not isnan(update[feat]), feat
          assert not isnan(self.weights[feat]), feat
          assert not isnan(prev)
          
          self.weight_norm -= abs(prev - eta*update[feat]) 
          self.weight_norm += abs(self.weights[feat])
          
          if self.weights[feat] == 0: del self.weights[feat]
      if USE_L2:
        for feat in update:
          prev = self.weights[feat]
          former = (prev - eta*update[feat]) ** 2 
          self.u -= (regularization_constant/self.N) * (1/(sigma*sigma)) * eta * self.weights[feat]
          self.weight_norm -= former 
          self.weight_norm += (self.weights[feat]) ** 2
          
        #self.dump(self.weights)
      print >>logs, "Weight Norm %s" % (self.weight_norm)
      #self.dump(self.weights)
      self.decoder.set_model_weights(self.weights)

    def compute_objective(self, log_likelihood):
      # loglikelihood = log p(y* | x ; w)
      objective = log_likelihood
      regular = 0.0
      if USE_L1 or USE_L2:
        regular = (regularization_constant / self.N) * self.weight_norm
        print >>logs, "Regularization = %s" % regular
        objective -= regular
      print >>logs, "Log-likelihood = %s" % log_likelihood

      print >>logs, "Estimate of objective = %s" % objective
      return objective, regular
    
    def one_pass_on_train(self):
      num_updates = 0
      train_fore = self.decoder.load(self.trainfiles)
      oracle_fore = self.decoder.load_oracle(self.oraclefiles)
      round_obj = []
      round_regular = []

      def collect_batch():
        update = Vector()
        cum_log_likelihood = 0.0
#.........这里部分代码省略.........

# 需要导入模块: from evaluator import Evaluator [as 别名]
# 或者: from evaluator.Evaluator import eval [as 别名]
class LBFGSCRF(trainer.BaseTrainer):

    def __init__(self, decoder, train, dev, output, iter = 1):
        trainer.BaseTrainer.__init__(self, decoder, [train], output=output)
        self.iter = iter
        self.eval = Evaluator(decoder, [dev])
        self.just_basic = False
        
    def set_oracle_files(self, oraclefiles):
      self.oraclefiles = oraclefiles

    def rm_features(self, remove_set):
      self.remove_set = remove_set

    def enforce_just_basic(self):
        self.just_basic = True

    def do_update(self, full_update, k):      
      print >>logs, "Doing update %s, Eta %s" % (k, eta)
      
      if DEBUG:
        all_feat = []
        for feat in full_update:
          all_feat.append((abs(full_update[feat]), full_update[feat], feat))
        all_feat.sort(reverse=True)
        self.dump(dict([(f,v2) for (v,v2,f) in all_feat[0:10]]))

      update = full_update
      self.weights += update

      if USE_L2:
        for feat in update:
          prev = self.weights[feat]
          former = (prev - eta*update[feat]) ** 2 
          self.u -= (regularization_constant/self.N) * (1/(sigma*sigma)) * eta * self.weights[feat]
          self.weight_norm -= former 
          self.weight_norm += (self.weights[feat]) ** 2
          
        #self.dump(self.weights)
      print >>logs, "Weight Norm %s" % (self.weight_norm)
      #self.dump(self.weights)
      self.decoder.set_model_weights(self.weights)


    def one_pass_on_train(self, weights):
      self.weights = weights
      for feat in weights:
          if feat.startswith("Basic"):
              print feat, weights[feat]
      self.decoder.set_model_weights(self.weights)
      self.round +=1 
      weight_file = open("tmp/weights.round."+str(self.round)+"."+str(self.name), 'w')
      for feat in self.weights:
        if abs(self.weights[feat]) > 1e-10:
          print >>weight_file, feat + "\t" + str(self.weights[feat])
      weight_file.close()
      #show train score
      #if self.round <> 1:
          #self.eval.tune()    
      prec = self.eval.eval()
      print "-----------------------"
      print "Final %s"%prec.compute_score()
      print "Num feat %s"%len(self.weights)
      print "-----------------------"

      try:
          num_updates = 0
          train_fore = self.decoder.load(self.trainfiles)
          oracle_fore = self.decoder.load_oracle(self.oraclefiles)

          update = Vector()
          cum_log_likelihood = 0.0
          start = time.time()
          for i, (example, oracle) in enumerate(izip(train_fore, oracle_fore), 1):
              if i == FLAGS.train_size:
                  break
              self.c += 1
              marginals, oracle_marginals, oracle_log_prob = self.decoder.compute_marginals(example, oracle)
              cum_log_likelihood += oracle_log_prob

              #little_update = (oracle_marginals - marginals)

              #print "Text Length", oracle_marginals["Basic/text-length"], marginals["Basic/text-length"], little_update["Basic/text-length"]
              #print "GT Prob", little_update["Basic/gt_prob"]

              update += oracle_marginals
              update -= marginals


              if i % 100 == 0:
                  end = time.time() 
                  print >> logs, "... example %d (len %d)...%s sec" % (i, len(example), (end-start)),
                   #print >> logs, "Oracle log prob %s"%oracle_log_prob
                  start = time.time()
      except:
          print "Unexpected error:", sys.exc_info()[0], traceback.print_exc(file=sys.stdout)  
          os._exit(0)
      #for i in range(len(self.ind2feature)):
      #  print i, update[self.ind2feature[i]], "\"" +self.ind2feature[i]+"\""

#.........这里部分代码省略.........

# 需要导入模块: from evaluator import Evaluator [as 别名]
# 或者: from evaluator.Evaluator import eval [as 别名]
 def test_falsy_eqv_special_form(self):
     e = Evaluator()
     exp = e.eval(read('(eqv? 1 2)'))
     self.assertEqual(exp, False)

# 需要导入模块: from evaluator import Evaluator [as 别名]
# 或者: from evaluator.Evaluator import eval [as 别名]

#.........这里部分代码省略.........
    flags.DEFINE_string("hadoop_weights", "", "hadoop weights (formatted specially)")
    flags.DEFINE_boolean("add_features", False, "add features to training data")
    flags.DEFINE_boolean("prune_train", False, "prune before decoding")
    flags.DEFINE_boolean("no_lm", False, "don't use the unigram language model")
    flags.DEFINE_boolean("pickleinput", False, "assumed input is pickled")
    flags.DEFINE_string("oracle_forests", None, "oracle forests", short_name="o")
    flags.DEFINE_string("feature_map_file", None, "file with the integer to feature mapping (for lbfgs)")
    flags.DEFINE_boolean("cache_input", False, "cache input sentences (only works for pruned input)")
    flags.DEFINE_string("rm_features", None, "list of features to remove")
    flags.DEFINE_boolean("just_basic", False, "remove all features but basic")

    argv = FLAGS(sys.argv)

    if FLAGS.weights:
        weights = Model.cmdline_model()
    else:
        vector = Vector()
        assert glob.glob(FLAGS.hadoop_weights)
        for file in glob.glob(FLAGS.hadoop_weights):
            for l in open(file):
                if not l.strip():
                    continue
                f, v = l.strip().split()
                vector[f] = float(v)
        weights = Model(vector)

    rm_features = set()
    if FLAGS.rm_features:
        for l in open(FLAGS.rm_features):
            rm_features.add(l.strip())

    lm = Ngram.cmdline_ngram()
    if FLAGS.no_lm:
        lm = None

    if argv[1] == "train":
        local_decode = ChiangPerceptronDecoder(weights, lm)
    elif argv[1] == "sgd" or argv[1] == "crf":
        local_decode = MarginalDecoder(weights, lm)
    else:
        local_decode = MarginalDecoder(weights, lm)

    if FLAGS.add_features:
        tdm = local_features.TargetDataManager()
        local_decode.feature_adder = FeatureAdder(tdm)
    local_decode.prune_train = FLAGS.prune_train
    local_decode.use_pickle = FLAGS.pickleinput
    local_decode.cache_input = FLAGS.cache_input
    print >> logs, "Cache input is %s" % FLAGS.cache_input
    if FLAGS.debuglevel > 0:
        print >> logs, "beam size = %d" % FLAGS.beam

    if argv[1] == "train":
        if not FLAGS.dist:
            perc = trainer.Perceptron.cmdline_perc(local_decode)
        else:
            train_files = [FLAGS.prefix + file.strip() for file in sys.stdin]
            perc = distributed_trainer.DistributedPerceptron.cmdline_perc(local_decode)
            perc.set_training(train_files)
        perc.train()
    elif argv[1] == "sgd":
        crf = sgd.BaseCRF.cmdline_crf(local_decode)
        crf.set_oracle_files([FLAGS.oracle_forests])
        crf.train()

    elif argv[1] == "crf":
        if not FLAGS.dist:
            crf = CRF.LBFGSCRF.cmdline_crf(local_decode)
            crf.set_oracle_files([FLAGS.oracle_forests])
            crf.set_feature_mappers(add_features.read_features(FLAGS.feature_map_file))
            crf.rm_features(rm_features)
            if FLAGS.just_basic:
                print "Enforcing Basic"
                crf.enforce_just_basic()
            crf.train()
        else:
            # train_files = [FLAGS.prefix+file.strip() for file in sys.stdin]
            # oracle_files = [file+".oracle" for file in train_files]
            print >> sys.stderr, "DistributedCRF"
            crf = distCRF.DistributedCRF.cmdline_distibuted_crf(local_decode)
            # os.system("~/.python/bin/dumbo rm train_input -hadoop /home/nlg-03/mt-apps/hadoop/0.20.1+169.89/")
            # os.system("~/.python/bin/dumbo put "+crf.trainfiles[0]+" train_input -hadoop /home/nlg-03/mt-apps/hadoop/0.20.1+169.89/")
            crf.set_feature_mappers(add_features.read_features(FLAGS.feature_map_file))
            crf.rm_features(rm_features)
            if FLAGS.just_basic:
                print "Enforcing Basic"
                crf.enforce_just_basic()

            # crf.set_oracle_files(oracle_files)
            crf.train()

    else:
        if not FLAGS.dist:
            print "Evaluating"
            eval = Evaluator(local_decode, [FLAGS.dev])
            eval.tune()
        else:
            dev_files = [FLAGS.prefix + file.strip() for file in sys.stdin]
            eval = Evaluator(local_decode, dev_files)
        print eval.eval(verbose=True).compute_score()

# 需要导入模块: from evaluator import Evaluator [as 别名]
# 或者: from evaluator.Evaluator import eval [as 别名]
 def test_complex_eqv_special_form(self):
     e = Evaluator()
     exp = e.eval(read('(let ((p (lambda (x) x))) (eqv? p p)'))
     self.assertEqual(exp, True)

# 需要导入模块: from evaluator import Evaluator [as 别名]
# 或者: from evaluator.Evaluator import eval [as 别名]
 def test_eqv_special_form(self):
     e = Evaluator()
     exp = e.eval(read('(eqv? 1 1)'))
     self.assertEqual(exp, True)

# 需要导入模块: from evaluator import Evaluator [as 别名]
# 或者: from evaluator.Evaluator import eval [as 别名]
 def test_define_func(self):
     e = Evaluator()
     exp = e.eval(read('(define (square a) (* a a))'))
     self.assertIsInstance(exp, type(lambda: None))

# 需要导入模块: from evaluator import Evaluator [as 别名]
# 或者: from evaluator.Evaluator import eval [as 别名]
 def test_define_lambda(self):
     e = Evaluator()
     exp = e.eval(read('(define square (lambda (x)  (* x x)))'))
     next_exp = e.eval(read('(square 5)'))
     self.assertEqual(next_exp, 25)

# 需要导入模块: from evaluator import Evaluator [as 别名]
# 或者: from evaluator.Evaluator import eval [as 别名]
 def test_define_var(self):
     e = Evaluator()
     exp = e.eval(read('(define x 4)'))
     self.assertEqual(e.env.get('x'), 4)