Python data.Data方法代码示例

# 需要导入模块: import data [as 别名]
# 或者: from data import Data [as 别名]
def __init__(self):
        super(Predictor, self).__init__()
        num_units = 512
        num_layer = 2
        batch_size = 1
        data_dir = 'data/'
        input_file = 'poetry.txt'
        vocab_file = 'vocab.pkl'
        tensor_file = 'tensor.npy'

        self.data = Data(data_dir, input_file, vocab_file, tensor_file, 
                        is_train=False, batch_size=batch_size)
        self.model = Net(self.data, num_units, num_layer, batch_size)
        self.sess = tf.Session()

        saver = tf.train.Saver(tf.global_variables())
        saver.restore(self.sess, 'model/model')
        print('Load model done.' + '\n') 

# 需要导入模块: import data [as 别名]
# 或者: from data import Data [as 别名]
def main():
    global model
    if args.data_test == ['video']:
        from videotester import VideoTester
        model = model.Model(args, checkpoint)
        t = VideoTester(args, model, checkpoint)
        t.test()
    else:
        if checkpoint.ok:
            loader = data.Data(args)
            _model = model.Model(args, checkpoint)
            _loss = loss.Loss(args, checkpoint) if not args.test_only else None
            t = Trainer(args, loader, _model, _loss, checkpoint)
            while not t.terminate():
                t.train()
                t.test()

            checkpoint.done() 

# 需要导入模块: import data [as 别名]
# 或者: from data import Data [as 别名]
def test():
    from data import Data
    from config import Config
    conf = Config()
    usecuda = True
    we = torch.load('./data/processed/ji/we.pkl')
    char_table = None
    sub_table = None
    if conf.need_char or conf.need_elmo:
        char_table = torch.load('./data/processed/ji/char_table.pkl')
    if conf.need_sub:
        sub_table = torch.load('./data/processed/ji/sub_table.pkl')
    model = IDRCModel(conf, we, char_table, sub_table, usecuda)
    if usecuda:
        model.cuda()
    d = Data(usecuda, conf)
    for a1, a2, sense, conn in d.train_loader:
        if usecuda:
            a1, a2 = a1.cuda(), a2.cuda()
        a1, a2 = Variable(a1), Variable(a2)
        break
    model.eval()
    out = model(a1, a2)
    print(out) 

# 需要导入模块: import data [as 别名]
# 或者: from data import Data [as 别名]
def createdata(path):
    """ Create training data by calling the Data class

    :type path: string
    :param path: path to the training document folder
    """
    data = Data()
    data.builddata(path)
    # Change the threshold if you want to filter
    #   out the low-frequency features
    data.buildvocab(thresh=1)
    data.buildmatrix()
    data.savematrix("training-data.pickle.gz")
    data.savevocab("vocab.pickle.gz") 

# 需要导入模块: import data [as 别名]
# 或者: from data import Data [as 别名]
def main():
    data = Data(dataname='kosarak', limit=20000)
    finder = SVSM(data, top_k=32, epsilon=4)
    cand_dict = finder.find()
    print(cand_dict) 

# 需要导入模块: import data [as 别名]
# 或者: from data import Data [as 别名]
def main():
    data = Data()
    finder = PEM(data, top_k=32, epsilon=4)
    cand_dict = finder.find()
    print(cand_dict) 

# 需要导入模块: import data [as 别名]
# 或者: from data import Data [as 别名]
def _pre_data(self):
        print('pre data...')
        self.data = Data(self.cuda, self.conf) 

# 需要导入模块: import data [as 别名]
# 或者: from data import Data [as 别名]
def update_pickle_file(file_name, eps=0, k=0, v=0):
    d_old = data_old.Data(file_name)
    d_old.load()
    print(file_name, 'loaded')
    # d_old.print_fields()

    d_new = data.Data()
    d_new.set_agent('Wolp',
                    int(d_old.get_data('max_actions')[0]),
                    k,
                    v)
    d_new.set_experiment(d_old.get_data('experiment')[0],
                         [-3],
                         [3],
                         eps)

    space = action_space.Space([-3], [3], int(d_old.get_data('max_actions')[0]))
    # print(space.get_space())
    # d_new.print_data()

    done = d_old.get_data('done')
    actors_result = d_old.get_data('actors_result')
    actions = d_old.get_data('actions')
    state_0 = d_old.get_data('state_0').tolist()
    state_1 = d_old.get_data('state_1').tolist()
    state_2 = d_old.get_data('state_2').tolist()
    state_3 = d_old.get_data('state_3').tolist()
    rewards = d_old.get_data('rewards').tolist()
    ep = 0
    temp = 0
    l = len(done)
    for i in range(l):
        d_new.set_action(space.import_point(actions[i]).tolist())
        d_new.set_actors_action(space.import_point(actors_result[i]).tolist())
        d_new.set_ndn_action(space.import_point(
            space.search_point(actors_result[i], 1)[0]).tolist())
        state = [state_0[i], state_1[i], state_2[i], state_3[i]]
        d_new.set_state(state)
        d_new.set_reward(1)
        if done[i] > 0:
            # print(ep, i - temp, 'progress', i / l)
            temp = i

            ep += 1
            # if ep % 200 == 199:
            #     d_new.finish_and_store_episode()
            # else:
            d_new.end_of_episode()

    d_new.save() 

# 需要导入模块: import data [as 别名]
# 或者: from data import Data [as 别名]
def run(args):

    save_dir = '{}/'.format(args.experiment_name)
    if not os.path.exists(save_dir):
        os.mkdir(save_dir)

    query = args.query
    k = args.k
    trained_prefix = args.trained_filename
    untrained_prefix = args.untrained_filename
    threshold = args.threshold

    search_space = Data('darts')

    # if it's the first iteration, choose k arches at random to train
    if query == 0:
        print('about to generate {} random'.format(k))
        data = search_space.generate_random_dataset(num=k, train=False)
        arches = [d['spec'] for d in data]

        next_arches = []
        for arch in arches:
            d = {}
            d['spec'] = arch
            next_arches.append(d)

    else:
        # get the data from prior iterations from pickle files
        data = []
        for i in range(query):

            filepath = '{}{}_{}.pkl'.format(save_dir, trained_prefix, i)
            with open(filepath, 'rb') as f:
                arch = pickle.load(f)
            data.append(arch)

        print('Iteration {}'.format(query))
        print('Data from last round')
        print(data)

        # run the meta neural net to output the next arches
        next_arches = run_meta_neuralnet(search_space, data, k=k)

    print('next batch')
    print(next_arches)

    # output the new arches to pickle files
    for i in range(k):
        index = query + i
        filepath = '{}{}_{}.pkl'.format(save_dir, untrained_prefix, index)
        next_arches[i]['index'] = index
        next_arches[i]['filepath'] = filepath
        with open(filepath, 'wb') as f:
            pickle.dump(next_arches[i], f) 

# 需要导入模块: import data [as 别名]
# 或者: from data import Data [as 别名]
def run_experiments(args, save_dir):

    os.environ['search_space'] = args.search_space

    from nas_algorithms import run_nas_algorithm
    from data import Data

    trials = args.trials
    out_file = args.output_filename
    save_specs = args.save_specs
    metann_params = meta_neuralnet_params(args.search_space)
    algorithm_params = algo_params(args.algo_params)
    num_algos = len(algorithm_params)
    logging.info(algorithm_params)

    # set up search space
    mp = copy.deepcopy(metann_params)
    ss = mp.pop('search_space')
    dataset = mp.pop('dataset')
    search_space = Data(ss, dataset=dataset)

    for i in range(trials):
        results = []
        walltimes = []
        run_data = []

        for j in range(num_algos):
            # run NAS algorithm
            print('\n* Running algorithm: {}'.format(algorithm_params[j]))
            starttime = time.time()
            algo_result, run_datum = run_nas_algorithm(algorithm_params[j], search_space, mp)
            algo_result = np.round(algo_result, 5)

            # remove unnecessary dict entries that take up space
            for d in run_datum:
                if not save_specs:
                    d.pop('spec')
                for key in ['encoding', 'adjacency', 'path', 'dist_to_min']:
                    if key in d:
                        d.pop(key)

            # add walltime, results, run_data
            walltimes.append(time.time()-starttime)
            results.append(algo_result)
            run_data.append(run_datum)

        # print and pickle results
        filename = os.path.join(save_dir, '{}_{}.pkl'.format(out_file, i))
        print('\n* Trial summary: (params, results, walltimes)')
        print(algorithm_params)
        print(metann_params)
        print(results)
        print(walltimes)
        print('\n* Saving to file {}'.format(filename))
        with open(filename, 'wb') as f:
            pickle.dump([algorithm_params, metann_params, results, walltimes, run_data], f)
            f.close() 

# 需要导入模块: import data [as 别名]
# 或者: from data import Data [as 别名]
def build_pdf(self, source, texinputs=[]):
        with TempDir() as tmpdir,\
                source.temp_saved(suffix='.latex', dir=tmpdir) as tmp:

            # close temp file, so other processes can access it also on Windows
            tmp.close()

            base_fn = os.path.splitext(tmp.name)[0]
            output_fn = base_fn + '.pdf'

            latex_cmd = [shlex_quote(self.pdflatex),
                         '-interaction=batchmode',
                         '-halt-on-error',
                         '-no-shell-escape',
                         '-file-line-error',
                         '%O',
                         '%S', ]

            if self.variant == 'pdflatex':
                args = [self.latexmk,
                        '-pdf',
                        '-pdflatex={}'.format(' '.join(latex_cmd)),
                        tmp.name, ]
            elif self.variant == 'xelatex':
                args = [self.latexmk,
                        '-xelatex',
                        tmp.name, ]
            else:
                raise ValueError('Invalid LaTeX variant: {}'.format(
                    self.variant))

            # create environment
            newenv = os.environ.copy()
            newenv['TEXINPUTS'] = os.pathsep.join(texinputs) + os.pathsep

            try:
                subprocess.check_call(args,
                                      cwd=tmpdir,
                                      env=newenv,
                                      stdin=open(os.devnull, 'r'),
                                      stdout=open(os.devnull, 'w'),
                                      stderr=open(os.devnull, 'w'), )
            except CalledProcessError as e:
                raise_from(LatexBuildError(base_fn + '.log'), e)

            return I(open(output_fn, 'rb').read(), encoding=None)