Python fluid.default_startup_program方法代码示例

# 需要导入模块: from paddle import fluid [as 别名]
# 或者: from paddle.fluid import default_startup_program [as 别名]
def __init__(self, model):
        super(StaticGraphAdapter, self).__init__()
        self.model = model
        # with `_build_once` gone, parameters are now created in `__init__`
        # so we need to keep track of the parameters already created
        self._startup_prog = fluid.default_startup_program()
        self._orig_prog = fluid.default_main_program()

        self._label_vars = {}  # label variables
        self._input_vars = {}  # label variables
        self._endpoints = {}
        self._loss_endpoint = None
        self._executor = None
        self._progs = {}
        self._compiled_progs = {}

        self._merge_count = {
            'eval_total': 0,
            'test_total': 0,
            'eval_batch': 0,
            'test_batch': 0
        }

        self._nranks = ParallelEnv().nranks
        self._local_rank = ParallelEnv().local_rank 

# 需要导入模块: from paddle import fluid [as 别名]
# 或者: from paddle.fluid import default_startup_program [as 别名]
def fit(self, dynamic):
        fluid.enable_dygraph(self.device) if dynamic else None
        seed = 333
        fluid.default_startup_program().random_seed = seed
        fluid.default_main_program().random_seed = seed

        model = LeNet()
        optim_new = fluid.optimizer.Adam(
            learning_rate=0.001, parameter_list=model.parameters())
        model.prepare(
            optim_new,
            loss_function=CrossEntropy(average=False),
            metrics=Accuracy(),
            inputs=self.inputs,
            labels=self.labels)
        model.fit(self.train_dataset, batch_size=64, shuffle=False)

        result = model.evaluate(self.val_dataset, batch_size=64)
        np.testing.assert_allclose(result['acc'], self.acc1)
        fluid.disable_dygraph() if dynamic else None 

# 需要导入模块: from paddle import fluid [as 别名]
# 或者: from paddle.fluid import default_startup_program [as 别名]
def train(save_dirname):
    x = fluid.layers.data(name='x', shape=[13], dtype='float32')
    y = fluid.layers.data(name='y', shape=[1], dtype='float32')
    y_predict, avg_cost = net(x, y)
    sgd_optimizer = fluid.optimizer.SGD(learning_rate=0.001)
    sgd_optimizer.minimize(avg_cost)
    train_reader = paddle.batch(
        paddle.reader.shuffle(paddle.dataset.uci_housing.train(), buf_size=500),
        batch_size=20)
    place = fluid.CPUPlace()
    exe = fluid.Executor(place)

    def train_loop(main_program):
        feeder = fluid.DataFeeder(place=place, feed_list=[x, y])
        exe.run(fluid.default_startup_program())

        PASS_NUM = 1000
        for pass_id in range(PASS_NUM):
            total_loss_pass = 0
            for data in train_reader():
                avg_loss_value, = exe.run(
                    main_program, feed=feeder.feed(data), fetch_list=[avg_cost])
                total_loss_pass += avg_loss_value
                if avg_loss_value < 5.0:
                    if save_dirname is not None:
                        fluid.io.save_inference_model(
                            save_dirname, ['x'], [y_predict], exe)
                    return
            print("Pass %d, total avg cost = %f" % (pass_id, total_loss_pass))

    train_loop(fluid.default_main_program())


# Infer by using provided test data. 

# 需要导入模块: from paddle import fluid [as 别名]
# 或者: from paddle.fluid import default_startup_program [as 别名]
def setUpClass(cls):
        cls.device = set_device('gpu')
        fluid.enable_dygraph(cls.device)

        sp_num = 1280
        cls.train_dataset = MnistDataset(mode='train', sample_num=sp_num)
        cls.val_dataset = MnistDataset(mode='test', sample_num=sp_num)
        cls.test_dataset = MnistDataset(
            mode='test', return_label=False, sample_num=sp_num)

        cls.train_loader = fluid.io.DataLoader(
            cls.train_dataset, places=cls.device, batch_size=64)
        cls.val_loader = fluid.io.DataLoader(
            cls.val_dataset, places=cls.device, batch_size=64)
        cls.test_loader = fluid.io.DataLoader(
            cls.test_dataset, places=cls.device, batch_size=64)

        seed = 333
        fluid.default_startup_program().random_seed = seed
        fluid.default_main_program().random_seed = seed

        dy_lenet = LeNetDygraph()
        cls.init_param = dy_lenet.state_dict()
        dynamic_train(dy_lenet, cls.train_loader)

        cls.acc1 = dynamic_evaluate(dy_lenet, cls.val_loader)

        cls.inputs = [Input([-1, 1, 28, 28], 'float32', name='image')]
        cls.labels = [Input([None, 1], 'int64', name='label')]

        cls.save_dir = tempfile.mkdtemp()
        cls.weight_path = os.path.join(cls.save_dir, 'lenet')
        fluid.dygraph.save_dygraph(dy_lenet.state_dict(), cls.weight_path)

        fluid.disable_dygraph() 

# 需要导入模块: from paddle import fluid [as 别名]
# 或者: from paddle.fluid import default_startup_program [as 别名]
def set_seed(self, seed=1024):
        fluid.default_startup_program().random_seed = seed
        fluid.default_main_program().random_seed = seed 

# 需要导入模块: from paddle import fluid [as 别名]
# 或者: from paddle.fluid import default_startup_program [as 别名]
def save_program():
    dataset = fluid.DatasetFactory().create_dataset()
    dataset.set_use_var(sparse_input_ids + [label])
    pipe_command = "python criteo_dataset.py {}".format(sys.argv[1])
    dataset.set_pipe_command(pipe_command)
    dataset.set_batch_size(32)
    dataset.set_thread(10)
    optimizer = fluid.optimizer.SGD(0.0001)
    optimizer.minimize(avg_cost)
    exe = fluid.Executor(fluid.CPUPlace())

    input_folder = "hdfs:"
    output = sp.check_output( "hadoop fs -D hadoop.job.ugi=" + hdfs_ugi
                              + " -D fs.defaultFS=" + hdfs_address +" -ls " + os.path.join(dataset_prefix, current_date_hr) + "/ | awk '{if(NR>1) print $8}'", shell=True)
    train_filelist = ["{}{}".format(input_folder, f) for f in output.decode('ascii').strip().split('\n')]
    train_filelist.remove('hdfs:' + os.path.join(dataset_prefix, current_date_hr, 'donefile'))
    train_filelist = [train_filelist[0]] 
    print(train_filelist)

    exe.run(fluid.default_startup_program())
    print("startup save program done.")
    dataset.set_filelist(train_filelist)
    exe.train_from_dataset(
        program=fluid.default_main_program(),
        dataset=dataset,
        fetch_list=[auc_var],
        fetch_info=["auc"],
        debug=False,)
        #print_period=10000)
    # save model here
    fetch_list = fluid.io.save_inference_model(inference_path, [x.name for x in inference_feed_vars], [predict], exe) 

# 需要导入模块: from paddle import fluid [as 别名]
# 或者: from paddle.fluid import default_startup_program [as 别名]
def do_train(args):
    device = set_device("gpu" if args.use_gpu else "cpu")
    fluid.enable_dygraph(device) if args.eager_run else None

    if args.enable_ce:
        fluid.default_main_program().random_seed = 102
        fluid.default_startup_program().random_seed = 102

    # define model
    inputs = [
        Input(
            [None, None], "int64", name="src_word"),
        Input(
            [None], "int64", name="src_length"),
        Input(
            [None, None], "int64", name="trg_word"),
    ]
    labels = [
        Input(
            [None], "int64", name="trg_length"),
        Input(
            [None, None, 1], "int64", name="label"),
    ]

    # def dataloader
    train_loader, eval_loader = create_data_loader(args, device)

    model_maker = get_model_cls(
        AttentionModel) if args.attention else get_model_cls(BaseModel)
    model = model_maker(args.src_vocab_size, args.tar_vocab_size,
                        args.hidden_size, args.hidden_size, args.num_layers,
                        args.dropout)
    grad_clip = fluid.clip.GradientClipByGlobalNorm(
        clip_norm=args.max_grad_norm)
    optimizer = fluid.optimizer.Adam(
        learning_rate=args.learning_rate,
        parameter_list=model.parameters(),
        grad_clip=grad_clip)

    ppl_metric = PPL(reset_freq=100)  # ppl for every 100 batches
    model.prepare(
        optimizer,
        CrossEntropyCriterion(),
        ppl_metric,
        inputs=inputs,
        labels=labels,
        device=device)
    model.fit(train_data=train_loader,
              eval_data=eval_loader,
              epochs=args.max_epoch,
              eval_freq=1,
              save_freq=1,
              save_dir=args.model_path,
              callbacks=[TrainCallback(ppl_metric, args.log_freq)]) 

# 需要导入模块: from paddle import fluid [as 别名]
# 或者: from paddle.fluid import default_startup_program [as 别名]
def test(model, args, im_shape):

    test_py_reader, prob, acc_1, acc_5 = build_program(args, False, model,
                                                       im_shape)

    test_prog = fluid.default_main_program().clone(for_test=True)

    place = fluid.CUDAPlace(0)
    exe = fluid.Executor(place)
    exe.run(fluid.default_startup_program())

    # yapf: disable
    if args.pretrained_model:
        def if_exist(var):
            return os.path.exists(os.path.join(args.pretrained_model, var.name))
        fluid.io.load_vars(exe, args.pretrained_model, predicate=if_exist)

    # yapf: enable

    exec_strategy = fluid.ExecutionStrategy()
    exec_strategy.num_threads = 1
    compile_program = fluid.compiler.CompiledProgram(
        test_prog).with_data_parallel(exec_strategy=exec_strategy)
    test_reader = reader.test10(args)
    test_py_reader.decorate_paddle_reader(test_reader)

    test_fetch_list = [prob, acc_1, acc_5]
    prob = []
    top1 = utils.AvgrageMeter()
    top5 = utils.AvgrageMeter()
    test_py_reader.start()
    test_start_time = time.time()
    step_id = 0
    try:
        while True:
            prev_test_start_time = test_start_time
            test_start_time = time.time()
            prob_v, acc_1_v, acc_5_v = exe.run(compile_program,
                                               test_prog,
                                               fetch_list=test_fetch_list)
            prob.append(list(np.array(prob_v)))
            top1.update(np.array(acc_1_v), np.array(prob_v).shape[0])
            top5.update(np.array(acc_5_v), np.array(prob_v).shape[0])
            if step_id % args.report_freq == 0:
                print('prob shape:', np.array(prob_v).shape)
                print("Step {}, acc_1 {}, acc_5 {}, time {}".format(
                    step_id,
                    np.array(acc_1_v),
                    np.array(acc_5_v), test_start_time - prev_test_start_time))
            step_id += 1
    except fluid.core.EOFException:
        test_py_reader.reset()
    np.concatenate(prob).dump(args.dump_path)
    print("top1 {0}, top5 {1}".format(top1.avg, top5.avg))