Python Progbar.add方法代码示例

# 需要导入模块: from keras.utils.generic_utils import Progbar [as 别名]
# 或者: from keras.utils.generic_utils.Progbar import add [as 别名]
    def learn(self, env, epoch=1, batch_size=1, exp_batch_size=0,
              gamma=0.9, reset_memory=False, verbose=1, callbacks=None):
        """Train Agent to play Enviroment env

        Parameters
        ----------
        env : :obj:`Enviroment`
            The enviroment the agent learn to play
        epoch : int
            number of complete episodes to play
        batch_size : int
            number of experiences to replay per step
        exp_batch_size : int
            number of experiences to replay from the consolidated
            :attr:`ExperienceReplayexperience.experience`.
        gamma : float
            discount factor
        reset_memory : bool
            if we should restart :attr:`ExperienceReplay.memory` before
            starting the game.
        verbose : int
            controls how much should we print
        callbacks : list of callables
            TODO: Add callback support

        """
        print("Learning started!")
        print("[Environment]: {}".format(env.description))
        print("[Model]: {}".format(self.model.description))
        print("[Memory]: {}".format(self.memory.description))
        if reset_memory:
            self.reset()
        progbar = Progbar(epoch)
        rewards = 0
        for e in xrange(epoch):
            # reset enviroment
            env.reset()
            game_over = False
            loss = 0

            # get initial observation, start game
            obs_t = env.observe()
            # Run an episonde
            while not game_over:
                obs_tm1 = obs_t
                action = self.policy(obs_tm1)

                # apply action, get rewards and new state
                obs_t, reward, game_over = env.update(action)
                rewards += reward

                # store experience
                self.remember(obs_tm1, action, reward, obs_t, game_over)

                # adapt model
                loss += self.update(batch_size=batch_size,
                                    exp_batch_size=exp_batch_size,
                                    gamma=gamma)
            if verbose == 1:
                progbar.add(1, values=[("loss", loss), ("rewards", rewards)])

# 需要导入模块: from keras.utils.generic_utils import Progbar [as 别名]
# 或者: from keras.utils.generic_utils.Progbar import add [as 别名]
def make_predictions(conf,shot_list,loader,custom_path=None):
    feature_extractor = FeatureExtractor(loader)
    save_prepath = feature_extractor.get_save_prepath()
    if custom_path == None:
        model_path = conf['paths']['model_save_path'] + model_filename#save_prepath + model_filename
    else:
        model_path = custom_path
    model = joblib.load(model_path)
    #shot_list = shot_list.random_sublist(10)

    y_prime = []
    y_gold = []
    disruptive = []

    pbar =  Progbar(len(shot_list))
    fn = partial(predict_single_shot,model=model,feature_extractor=feature_extractor)
    pool = mp.Pool()
    print('predicting in parallel on {} processes'.format(pool._processes))
    #for (y_p,y,disr) in map(fn,shot_list):
    for (y_p,y,disr) in pool.imap(fn,shot_list):
        #y_p,y,disr = predict_single_shot(model,feature_extractor,shot)
        y_prime += [np.expand_dims(y_p,axis=1)]
        y_gold += [np.expand_dims(y,axis=1)]
        disruptive += [disr]
        pbar.add(1.0)

    pool.close()
    pool.join()
    return y_prime,y_gold,disruptive

# 需要导入模块: from keras.utils.generic_utils import Progbar [as 别名]
# 或者: from keras.utils.generic_utils.Progbar import add [as 别名]
    def play(self, env, epoch=1, batch_size=1, visualize=None, verbose=1):
        print("Free play started!")
        frames = np.zeros((0, ) + env.observe_image().shape[1:])
        frames = frames.transpose(0, 2, 3, 1)
        rewards = 0
        progbar = Progbar(epoch)

        for e in xrange(epoch):
            env.reset()
            game_over = False
            loss = 0
            # get initial observation, start game
            obs_t = env.observe()
            while not game_over:
                obs_tm1 = obs_t

                # get next action
                action = self.policy(obs_tm1, train=False)

                # apply action, get rewareds and new state
                obs_t, reward, game_over = env.update(action)
                rewards += reward

                frame_t = env.observe_image().transpose(0, 2, 3, 1)
                frames = np.concatenate([frames, frame_t], axis=0)

            if verbose == 1:
                progbar.add(1, values=[("loss", loss), ("rewards", rewards)])

        if visualize:
            print("Making gif!")
            frames = np.repeat(frames, 3, axis=-1)
            make_gif(frames[:-visualize['n_frames']],
                     filepath=visualize['filepath'], gray=visualize['gray'])
            print("See your gif at {}".format(visualize['filepath']))

# 需要导入模块: from keras.utils.generic_utils import Progbar [as 别名]
# 或者: from keras.utils.generic_utils.Progbar import add [as 别名]
def validate(dev, gen_test, beam_size, hypo_len, samples, noise_size, glove, cmodel = None, adverse = False, 
                 diverse = False):
    vgen = val_generator(dev, gen_test, beam_size, hypo_len, noise_size)
    p = Progbar(samples)
    batchez = []
    while p.seen_so_far < samples:
        batch = next(vgen)
        preplexity = np.mean(np.power(2, batch[2]))
        loss = np.mean(batch[2])
        losses = [('hypo_loss',loss),('perplexity', preplexity)]
        if cmodel is not None:
            ceval = cmodel.evaluate([batch[0], batch[1]], batch[4], verbose = 0)
            losses += [('class_loss', ceval[0]), ('class_acc', ceval[1])]
            probs = cmodel.predict([batch[0], batch[1]], verbose = 0)
            losses += [('class_entropy', np.mean(-np.sum(probs * np.log(probs), axis=1)))]
        
        p.add(len(batch[0]), losses)
        batchez.append(batch)
    batchez = merge_result_batches(batchez)
    
    res = {}
    if adverse:
        val_loss = adverse_validation(dev, batchez, glove)
        print 'adverse_loss:', val_loss
        res['adverse_loss'] = val_loss
    if diverse:
        div, _, _, _ = diversity(dev, gen_test, beam_size, hypo_len, noise_size, 64, 32)
        res['diversity'] = div
    print
    for val in p.unique_values:
        arr = p.sum_values[val]
        res[val] = arr[0] / arr[1]
    return res

# 需要导入模块: from keras.utils.generic_utils import Progbar [as 别名]
# 或者: from keras.utils.generic_utils.Progbar import add [as 别名]
def adverse_generate2(gen_model, ad_model, cmodel, train, word_index, glove, threshold = 0.95, batch_size = 64, ci = False):
    mb = load_data.get_minibatches_idx(len(train), batch_size, shuffle=True)
    p = Progbar(len(train))
    results = []
    for i, train_index in mb:
        if len(train_index) != batch_size:
            continue
        orig_batch = [train[k] for k in train_index]
        class_indices = [load_data.LABEL_LIST.index(train[k][2]) for k in train_index]
        probs = generation.generation_predict_embed(gen_model, word_index.index, orig_batch,
                     np.random.random_integers(0, len(train), len(orig_batch)), class_indices = class_indices)
        gen_batch = generation.get_classes(probs)
        ad_preds = ad_model.predict_on_batch(gen_batch)[0].flatten()
        
        X = []
        for i in range(len(orig_batch)):
	    concat = orig_batch[i][0] + ["--"] + word_index.get_seq(gen_batch[i])
            X.append(load_data.load_word_vecs(concat, glove))
        X = np.array(X)
        X_padded = load_data.pad_sequences(X, dim = len(X[0][0]))
        cpreds = cmodel.predict_on_batch(X_padded)[0][np.arange(len(X_padded)), class_indices]
        
        pred_seq = [word_index.print_seq(gen) for gen in gen_batch]
        premises = [" ".join(ex[0]) for ex in orig_batch]
        classes = np.array(load_data.LABEL_LIST)[class_indices]
        zipped = zip(cpreds, ad_preds, premises, pred_seq, classes)
        results += [el for el in zipped if el[0] * el[1]> threshold]
        p.add(len(train_index),[('added', float(len([el for el in zipped if el[0] * el[1]> threshold])))])
        if len(results) > 200:
            print (i + 1) * batch_size
            return results
    return results

# 需要导入模块: from keras.utils.generic_utils import Progbar [as 别名]
# 或者: from keras.utils.generic_utils.Progbar import add [as 别名]
def train_model_embed(train, dev, glove, model, model_dir = 'models/curr_model', nb_epochs = 20, batch_size = 64, hs=True, ci = True):
    X_dev_p, X_dev_h, y_dev = load_data.prepare_split_vec_dataset(dev, glove=glove)
    
    word_index = load_data.WordIndex(glove)
    if not os.path.exists(model_dir):
         os.makedirs(model_dir)
    for e in range(nb_epochs):
        print "Epoch ", e
        mb = load_data.get_minibatches_idx(len(train), batch_size, shuffle=True)
        p = Progbar(len(train))
        for i, train_index in mb:
            if len(train_index) != batch_size:
                continue
            X_train_p, X_train_h , y_train = load_data.prepare_split_vec_dataset([train[k] for k in train_index], word_index.index)
            padded_p = load_data.pad_sequences(X_train_p, maxlen = PREM_LEN, dim = -1, padding = 'pre')
            padded_h = load_data.pad_sequences(X_train_h, maxlen = HYPO_LEN, dim = -1, padding = 'post')
            
            data = {'premise_input': padded_p, 'embed_input': np.expand_dims(np.array(train_index), axis=1), 'output' : padded_h}
            if ci:
                data['class_input'] = y_train
            if hs:
                data['train_input'] = padded_h
                data['output'] = np.ones((batch_size, HYPO_LEN, 1))
            
            #sw = (padded_h != 0).astype(float)
            #train_loss = float(model.train_on_batch(data, sample_weight={'output':sw})[0])
	    train_loss = float(model.train_on_batch(data)[0])
            p.add(len(train_index),[('train_loss', train_loss)])
        sys.stdout.write('\n')
        model.save_weights(model_dir + '/model~' + str(e))

# 需要导入模块: from keras.utils.generic_utils import Progbar [as 别名]
# 或者: from keras.utils.generic_utils.Progbar import add [as 别名]
    def run_epoch(self, split, train=False, batch_size=128, return_pred=False):
        total = total_loss = 0
        func = self.model.train_on_batch if train else self.model.test_on_batch
        ids, preds, targs = [], [], []
        prog = Progbar(split.num_examples)
        for idx, X, Y, types in split.batches(batch_size):
            X.update({k: np.concatenate([v, types], axis=1) for k, v in Y.items()})
            batch_end = time()
            loss = func(X)
            prob = self.model.predict(X, verbose=0)['p_relation']
            prob *= self.typechecker.get_valid_cpu(types[:, 0], types[:, 1])
            pred = prob.argmax(axis=1)

            targ = Y['p_relation'].argmax(axis=1)
            ids.append(idx)
            targs.append(targ)
            preds.append(pred)
            total_loss += loss
            total += 1
            prog.add(idx.size, values=[('loss', loss), ('acc', np.mean(pred==targ))])
        preds = np.concatenate(preds).astype('int32')
        targs = np.concatenate(targs).astype('int32')
        ids = np.concatenate(ids).astype('int32')

        ret = {
            'f1': f1_score(targs, preds, average='micro', labels=self.labels),
            'precision': precision_score(targs, preds, average='micro', labels=self.labels),
            'recall': recall_score(targs, preds, average='micro', labels=self.labels),
            'accuracy': accuracy_score(targs, preds),
            'loss': total_loss / float(total),
            }
        if return_pred:
            ret.update({'ids': ids.tolist(), 'preds': preds.tolist(), 'targs': targs.tolist()})
        return ret

# 需要导入模块: from keras.utils.generic_utils import Progbar [as 别名]
# 或者: from keras.utils.generic_utils.Progbar import add [as 别名]
def preprocess(X):
    progbar = Progbar(X.shape[0])  # progress bar for pre-processing status tracking

    for i in range(X.shape[0]):
        for j in range(X.shape[1]):
            X[i, j] = denoise_tv_chambolle(X[i, j], weight=0.1, multichannel=False)
        progbar.add(1)
    return X

# 需要导入模块: from keras.utils.generic_utils import Progbar [as 别名]
# 或者: from keras.utils.generic_utils.Progbar import add [as 别名]
def generation_test(train, glove, model, batch_size = 64, prem_len = 22, hypo_len = 12):
    mb = load_data.get_minibatches_idx(len(train), batch_size, shuffle=True)
    p = Progbar(len(train))
    for i, train_index in mb:
        X_prem, X_hypo, _ = load_data.prepare_split_vec_dataset([train[k] for k in train_index], glove)
        X_p = load_data.pad_sequences(X_prem, maxlen = prem_len, dim = 50)
        X_h = load_data.pad_sequences(X_hypo, maxlen = hypo_len, dim = 50)
        train_loss =  model.train_on_batch(X_p, X_h)[0]
        p.add(len(X_p),[('train_loss', train_loss)])

# 需要导入模块: from keras.utils.generic_utils import Progbar [as 别名]
# 或者: from keras.utils.generic_utils.Progbar import add [as 别名]
def rotation_augmentation(X, angle_range):
    progbar = Progbar(X.shape[0]) 
    X_rot = np.copy(X)
    for i in range(len(X)):
        angle = np.random.randint(-angle_range, angle_range)
        for j in range(X.shape[1]):
            X_rot[i, j] = ndimage.rotate(X[i, j], angle, reshape=False, order=1)
        progbar.add(1)
    return X_rot

# 需要导入模块: from keras.utils.generic_utils import Progbar [as 别名]
# 或者: from keras.utils.generic_utils.Progbar import add [as 别名]
def preprocess(X):
    "Pre-process images that are fed to neural network"
    progbar = Progbar(X.shape[0])  # progress bar for pre-processing status tracking

    for i in range(X.shape[0]):
        for j in range(X.shape[1]):
            X[i, j] = denoise_tv_chambolle(X[i, j], weight=0.1, multichannel=False)
        progbar.add(1)
    return X		# Denoising weight is the regularization parameter

# 需要导入模块: from keras.utils.generic_utils import Progbar [as 别名]
# 或者: from keras.utils.generic_utils.Progbar import add [as 别名]
def batchwise_function(func, X, batch_size=100, verbose=1):
    # Y = [func([X[i*batch_size:(i+1)*batch_size]]) for i in range(
    #    0, X.shape[0]//batch_size)]
    Y = []
    progbar = Progbar(X.shape[0])
    for i in range(0, X.shape[0] // batch_size):
        Y += [func([X[i*batch_size:(i+1)*batch_size]])]
        if verbose > 0:
            progbar.add(batch_size)
    return np.concatenate(Y, axis=0)

# 需要导入模块: from keras.utils.generic_utils import Progbar [as 别名]
# 或者: from keras.utils.generic_utils.Progbar import add [as 别名]
def zoom_augmentation(X, y, k_min):
    progbar = Progbar(X.shape[0])  # progress bar for augmentation status tracking
    X_zoom = np.copy(X)
    y_zoom = np.copy(y)
    for i in range(len(X)):
        k_random = 1.0 - (np.random.rand() * (1.0 - k_min))
        for j in range(X.shape[1]):
            X_zoom[i, j] = zoom(X[i, j], k_random)
        y_zoom[i] *= 1 / (k_random * k_random)
        progbar.add(1)
    return X_zoom, y_zoom

# 需要导入模块: from keras.utils.generic_utils import Progbar [as 别名]
# 或者: from keras.utils.generic_utils.Progbar import add [as 别名]
def main():
    corpora = argv[1:]
    corpora = [x[:-1] if x.endswith(os.sep) else x for x in corpora]
    num_files = sum([len(os.listdir(x)) for x in corpora])
    pb = Progbar(num_files)
    for corpus in corpora:
        mkdir_p(corpus + '_downsampled')
        for filename in os.listdir(corpus):
            if filename.endswith('wav'):
                downsample(os.path.join(corpus, filename), os.path.join(corpus + '_downsampled', filename), verbose=False)
            pb.add(1)

# 需要导入模块: from keras.utils.generic_utils import Progbar [as 别名]
# 或者: from keras.utils.generic_utils.Progbar import add [as 别名]
def sampling_augmentation(X, n):
    progbar = Progbar(X.shape[0])

    X_sampled = []
    for i in range(len(X)):
        slices = np.copy(X[i])
        ix = np.random.choice(range(len(slices)), n, replace=False)
        np.random.shuffle(ix)
        X_sampled.append(slices[ix,])
        progbar.add(1)
    return np.array(X_sampled)