C++ NumericalDifferentiation类代码示例

void NumericalDifferentiationTest::test_calculate_gradient(void)
{
   message += "test_calculate_gradient\n";

   NumericalDifferentiation nd;

   Vector<double> x;
   Vector<double> g;
	   
   // Test

   nd.set_numerical_differentiation_method(NumericalDifferentiation::ForwardDifferences);

   x.set(2, 0.0);

   g = nd.calculate_gradient(*this, &NumericalDifferentiationTest::f2, x);

   assert_true(g.size() == 2, LOG);
   assert_true(g == 1.0, LOG);

   // Test

   nd.set_numerical_differentiation_method(NumericalDifferentiation::CentralDifferences);

   x.set(2, 0.0);

   g = nd.calculate_gradient(*this, &NumericalDifferentiationTest::f2, x);

   assert_true(g.size() == 2, LOG);
   assert_true(g == 1.0, LOG);
}

void ProbabilisticLayerTest::test_calculate_Jacobian(void)
{
   message += "test_calculate_Jacobian\n";

   NumericalDifferentiation nd;

   ProbabilisticLayer pl;

   Vector<double> inputs;
   Matrix<double> Jacobian;
   Matrix<double> numerical_Jacobian;

   // Test

   if(numerical_differentiation_tests)
   {
      pl.set_probabilistic_method(ProbabilisticLayer::Softmax);

      pl.set(3);

      inputs.set(3);
      inputs.randomize_normal();

      Jacobian = pl.calculate_Jacobian(inputs);
      numerical_Jacobian = nd.calculate_Jacobian(pl, &ProbabilisticLayer::calculate_outputs, inputs);

      assert_true((Jacobian-numerical_Jacobian).calculate_absolute_value() < 1.0e-3, LOG);
   }
}

void NormalizedSquaredErrorTest::test_calculate_gradient(void)
{
   message += "test_calculate_gradient\n";

   NumericalDifferentiation nd;

   NeuralNetwork nn;

   Vector<double> network_parameters;

   DataSet ds;
   Matrix<double> data;

   NormalizedSquaredError nse(&nn, &ds);

   Vector<double> objective_gradient;
   Vector<double> numerical_objective_gradient;

   // Test 

   nn.set(1,1,1);

   nn.initialize_parameters(0.0);

   ds.set(1, 1, 2);

   data.set(2, 2);
   data[0][0] = -1.0;
   data[0][1] = -1.0;
   data[1][0] = 1.0;
   data[1][1] = 1.0;

   ds.set_data(data);

   objective_gradient = nse.calculate_gradient();

   assert_true(objective_gradient.size() == nn.count_parameters_number(), LOG);
   assert_true(objective_gradient == 0.0, LOG);

   // Test 

   nn.set(3, 4, 5);
   nn.randomize_parameters_normal();

   network_parameters = nn.arrange_parameters();

   ds.set(3, 5, 2);
   ds.randomize_data_normal();

   objective_gradient = nse.calculate_gradient();
   numerical_objective_gradient = nd.calculate_gradient(nse, &NormalizedSquaredError::calculate_performance, network_parameters);

   assert_true((objective_gradient - numerical_objective_gradient).calculate_absolute_value() < 1.0e-3, LOG);
}

void RootMeanSquaredErrorTest::test_calculate_gradient(void)
{
   message += "test_calculate_gradient\n";

   NumericalDifferentiation nd;

   NeuralNetwork nn;

   Vector<double> network_parameters;

   DataSet ds;

   RootMeanSquaredError rmse(&nn, &ds);

   Vector<double> objective_gradient;
   Vector<double> numerical_objective_gradient;

   // Test

   nn.set(3, 4, 2);
   nn.initialize_parameters(0.0);

   ds.set(3, 2, 5);
   ds.initialize_data(0.0);

   // Test

   nn.set(3, 4, 2);
   nn.initialize_parameters(1.0);

   network_parameters = nn.arrange_parameters();

   ds.set(3, 2, 5);
   ds.initialize_data(1.0);

   objective_gradient = rmse.calculate_gradient();
   numerical_objective_gradient = nd.calculate_gradient(rmse, &RootMeanSquaredError::calculate_performance, network_parameters);
   assert_true((objective_gradient - numerical_objective_gradient).calculate_absolute_value() < 1.0e-3, LOG);

   // Test

   nn.set(1,1,1);

   network_parameters = nn.arrange_parameters();

   ds.set(1,1,1);
   ds.initialize_data(1.0);

   rmse.set_neural_network_pointer(&nn);

   objective_gradient = rmse.calculate_gradient();
   numerical_objective_gradient = nd.calculate_gradient(rmse, &RootMeanSquaredError::calculate_performance, network_parameters);
   assert_true((objective_gradient - numerical_objective_gradient).calculate_absolute_value() < 1.0e-3, LOG);
}

void NumericalDifferentiationTest::test_calculate_central_differences_second_derivative(void)
{
   message += "test_calculate_central_differences_second_derivative\n";

   NumericalDifferentiation nd;

   double x;
   double d2;

   // Test

   x = 0.0;
   d2 = nd.calculate_central_differences_second_derivative(*this, &NumericalDifferentiationTest::f1, x);

   assert_true(fabs(d2) <= 1.0e-6, LOG);
}

void NumericalDifferentiationTest::test_calculate_central_differences_derivative(void)
{
   message += "test_calculate_central_differences_derivative\n";

   NumericalDifferentiation nd;

   double x;
   double d;

   // Test

   x = 0.0;
   d = nd.calculate_central_differences_derivative(*this, &NumericalDifferentiationTest::f1, x);

   assert_true(d == 1.0, LOG);
}

void NumericalDifferentiationTest::test_calculate_forward_differences_gradient(void)
{
   message += "test_calculate_forward_differences_gradient\n";

   NumericalDifferentiation nd;

   Vector<double> x;
   Vector<double> g;
	   
   // Test

   x.set(2, 0.0);

   g = nd.calculate_forward_differences_gradient(*this, &NumericalDifferentiationTest::f2, x);

   assert_true(g.size() == 2, LOG);
   assert_true(g == 1.0, LOG);
}

void NumericalDifferentiationTest::test_calculate_central_differences_Hessian_form(void)
{
   message += "test_calculate_central_differences_Hessian_form\n";

   NumericalDifferentiation nd;

   Vector<double> x(2, 0.0);

   Vector< Matrix<double> > Hessian = nd.calculate_central_differences_Hessian_form(*this, &NumericalDifferentiationTest::f3, x);

   assert_true(Hessian.size() == 2, LOG);

   assert_true(Hessian[0].get_rows_number() == 2, LOG);
   assert_true(Hessian[0].get_columns_number() == 2, LOG);
   assert_true(Hessian[0] == 0.0, LOG);

   assert_true(Hessian[1].get_rows_number() == 2, LOG);
   assert_true(Hessian[1].get_columns_number() == 2, LOG);
   assert_true(Hessian[1] == 0.0, LOG);
}

void NumericalDifferentiationTest::test_calculate_forward_differences_Jacobian(void)
{
   message += "test_calculate_forward_differences_Jacobian\n";

   NumericalDifferentiation nd;

   Vector<double> x;
   Matrix<double> J;

   Matrix<double> J_true;

   // Test

   x.set(2, 0.0);

   J = nd.calculate_forward_differences_Jacobian(*this, &NumericalDifferentiationTest::f3, x);

   J_true.set(2, 2);
   J_true.initialize_identity();

   assert_true(J == J_true, LOG);
}

void MeanSquaredErrorTest::test_calculate_Jacobian_terms(void)
{
   message += "test_calculate_Jacobian_terms\n";

   NumericalDifferentiation nd;

   NeuralNetwork nn;
   Vector<unsigned> multilayer_perceptron_architecture;
   Vector<double> parameters;

   DataSet ds;

   MeanSquaredError mse(&nn, &ds);

   Vector<double> objective_gradient;

   Vector<double> evaluation_terms;
   Matrix<double> terms_Jacobian;
   Matrix<double> numerical_Jacobian_terms;

   // Test

   nn.set(1, 1);

   nn.initialize_parameters(0.0);

   ds.set(1, 1, 1);

   ds.initialize_data(0.0);

   terms_Jacobian = mse.calculate_terms_Jacobian();

   assert_true(terms_Jacobian.get_rows_number() == ds.get_instances().count_training_instances_number(), LOG);
   assert_true(terms_Jacobian.get_columns_number() == nn.count_parameters_number(), LOG);
   assert_true(terms_Jacobian == 0.0, LOG);

   // Test 

   nn.set(3, 4, 2);
   nn.initialize_parameters(0.0);

   ds.set(3, 2, 5);
   mse.set(&nn, &ds);
   ds.initialize_data(0.0);

   terms_Jacobian = mse.calculate_terms_Jacobian();

   assert_true(terms_Jacobian.get_rows_number() == ds.get_instances().count_training_instances_number(), LOG);
   assert_true(terms_Jacobian.get_columns_number() == nn.count_parameters_number(), LOG);
   assert_true(terms_Jacobian == 0.0, LOG);

   // Test

   multilayer_perceptron_architecture.set(3);
   multilayer_perceptron_architecture[0] = 2;
   multilayer_perceptron_architecture[1] = 1;
   multilayer_perceptron_architecture[2] = 2;

   nn.set(multilayer_perceptron_architecture);
   nn.initialize_parameters(0.0);

   ds.set(2, 2, 5);
   mse.set(&nn, &ds);
   ds.initialize_data(0.0);

   terms_Jacobian = mse.calculate_terms_Jacobian();

   assert_true(terms_Jacobian.get_rows_number() == ds.get_instances().count_training_instances_number(), LOG);
   assert_true(terms_Jacobian.get_columns_number() == nn.count_parameters_number(), LOG);
   assert_true(terms_Jacobian == 0.0, LOG);

   // Test

   nn.set(1, 1, 1);
   nn.randomize_parameters_normal();
   parameters = nn.arrange_parameters();

   ds.set(1, 1, 1);
   ds.randomize_data_normal();

   terms_Jacobian = mse.calculate_terms_Jacobian();
   numerical_Jacobian_terms = nd.calculate_Jacobian(mse, &MeanSquaredError::calculate_terms, parameters);

   assert_true((terms_Jacobian-numerical_Jacobian_terms).calculate_absolute_value() < 1.0e-3, LOG);

   // Test

   nn.set(2, 2, 2);
   nn.randomize_parameters_normal();
   parameters = nn.arrange_parameters();

   ds.set(2, 2, 2);
   ds.randomize_data_normal();

   terms_Jacobian = mse.calculate_terms_Jacobian();
   numerical_Jacobian_terms = nd.calculate_Jacobian(mse, &MeanSquaredError::calculate_terms, parameters);

   assert_true((terms_Jacobian-numerical_Jacobian_terms).calculate_absolute_value() < 1.0e-3, LOG);

   // Test
//.........这里部分代码省略.........

void MeanSquaredErrorTest::test_calculate_gradient(void)
{
   message += "test_calculate_gradient\n";

   NumericalDifferentiation nd;

   NeuralNetwork nn;
   Vector<unsigned> multilayer_perceptron_architecture;

   Vector<double> parameters;

   DataSet ds;

   MeanSquaredError mse(&nn, &ds);

   Vector<double> objective_gradient;
   Vector<double> numerical_objective_gradient;
   Vector<double> numerical_differentiation_error;

   // Test

   nn.set(1, 1, 1);

   nn.initialize_parameters(0.0);

   ds.set(1, 1, 1);

   ds.initialize_data(0.0);

   objective_gradient = mse.calculate_gradient();

   assert_true(objective_gradient.size() == nn.count_parameters_number(), LOG);
   assert_true(objective_gradient == 0.0, LOG);

   // Test 

   nn.set(3, 4, 2);
   nn.initialize_parameters(0.0);

   ds.set(3, 2, 5);
   mse.set(&nn, &ds);
   ds.initialize_data(0.0);

   objective_gradient = mse.calculate_gradient();

   assert_true(objective_gradient.size() == nn.count_parameters_number(), LOG);
   assert_true(objective_gradient == 0.0, LOG);

   // Test

   multilayer_perceptron_architecture.set(3);
   multilayer_perceptron_architecture[0] = 2;
   multilayer_perceptron_architecture[1] = 1;
   multilayer_perceptron_architecture[2] = 3;

   nn.set(multilayer_perceptron_architecture);
   nn.initialize_parameters(0.0);

   ds.set(2, 3, 5);
   mse.set(&nn, &ds);
   ds.initialize_data(0.0);

   objective_gradient = mse.calculate_gradient();

   assert_true(objective_gradient.size() == nn.count_parameters_number(), LOG);
   assert_true(objective_gradient == 0.0, LOG);

   // Test

   nn.set(1, 1, 1);

   nn.initialize_parameters(0.0);

   ds.set(1, 1, 1);

   ds.initialize_data(0.0);

   objective_gradient = mse.calculate_gradient();

   assert_true(objective_gradient.size() == nn.count_parameters_number(), LOG);
   assert_true(objective_gradient == 0.0, LOG);

   // Test 

   nn.set(3, 4, 2);
   nn.initialize_parameters(0.0);

   ds.set(3, 2, 5);
   mse.set(&nn, &ds);
   ds.initialize_data(0.0);

   objective_gradient = mse.calculate_gradient();

   assert_true(objective_gradient.size() == nn.count_parameters_number(), LOG);
   assert_true(objective_gradient == 0.0, LOG);

   // Test

   nn.set(1, 1);
   nn.initialize_parameters(1.0);
//.........这里部分代码省略.........

void NeuralParametersNormTest::test_calculate_gradient(void)
{
   message += "test_calculate_gradient\n";

   NumericalDifferentiation nd;
   NeuralNetwork nn;
   NeuralParametersNorm npn(&nn);

   Vector<size_t> architecture;

   Vector<double> parameters;
   Vector<double> gradient;
   Vector<double> numerical_gradient;
   Vector<double> error;

   // Test 

   nn.set(1, 1, 1);
   nn.initialize_parameters(0.0);

   gradient = npn.calculate_gradient();

   assert_true(gradient.size() == nn.count_parameters_number(), LOG);
   assert_true(gradient == 0.0, LOG);

   // Test 

   nn.set(3, 4, 2);
   nn.initialize_parameters(0.0);

   gradient = npn.calculate_gradient();

   assert_true(gradient.size() == nn.count_parameters_number(), LOG);
   assert_true(gradient == 0.0, LOG);

   // Test

   architecture.set(3);
   architecture[0] = 5;
   architecture[1] = 1;
   architecture[2] = 2;

   nn.set(architecture);
   nn.initialize_parameters(0.0);

   npn.set_neural_network_pointer(&nn);

   gradient = npn.calculate_gradient();

   assert_true(gradient.size() == nn.count_parameters_number(), LOG);
   assert_true(gradient == 0.0, LOG);

   // Test 

   nn.set(3, 4, 2);
   nn.initialize_parameters(0.0);

   npn.set_neural_network_pointer(&nn);

   gradient = npn.calculate_gradient();

   assert_true(gradient.size() == nn.count_parameters_number(), LOG);
   assert_true(gradient == 0.0, LOG);


   // Test

   nn.initialize_parameters(1.0);
   parameters = nn.arrange_parameters();

   gradient = npn.calculate_gradient();
   numerical_gradient = nd.calculate_gradient(npn, &NeuralParametersNorm::calculate_regularization, parameters);
   error = (gradient - numerical_gradient).calculate_absolute_value();

   assert_true(error < 1.0e-3, LOG);
}

void NormalizedSquaredErrorTest::test_calculate_Jacobian_terms(void)
{
   message += "test_calculate_Jacobian_terms\n";

   NumericalDifferentiation nd;

   NeuralNetwork nn;
   Vector<int> hidden_layers_size;
   Vector<double> network_parameters;

   DataSet ds;

   NormalizedSquaredError nse(&nn, &ds);

   Vector<double> objective_gradient;

   Vector<double> evaluation_terms;
   Matrix<double> terms_Jacobian;
   Matrix<double> numerical_Jacobian_terms;

   // Test

   nn.set(1, 1);
   nn.randomize_parameters_normal();
   network_parameters = nn.arrange_parameters();

   ds.set(1, 1, 2);
   ds.randomize_data_normal();

   terms_Jacobian = nse.calculate_terms_Jacobian();
   numerical_Jacobian_terms = nd.calculate_Jacobian(nse, &NormalizedSquaredError::calculate_terms, network_parameters);

   assert_true((terms_Jacobian-numerical_Jacobian_terms).calculate_absolute_value() < 1.0e-3, LOG);

   // Test

   nn.set(2, 2, 2);
   nn.randomize_parameters_normal();
   network_parameters = nn.arrange_parameters();

   ds.set(2, 2, 2);
   ds.randomize_data_normal();

   terms_Jacobian = nse.calculate_terms_Jacobian();
   numerical_Jacobian_terms = nd.calculate_Jacobian(nse, &NormalizedSquaredError::calculate_terms, network_parameters);

   assert_true((terms_Jacobian-numerical_Jacobian_terms).calculate_absolute_value() < 1.0e-3, LOG);

   // Test

   nn.set(2,2,2);
   nn.randomize_parameters_normal();

   ds.set(2,2,2);
   ds.randomize_data_normal();
   
   objective_gradient = nse.calculate_gradient();

   evaluation_terms = nse.calculate_terms();
   terms_Jacobian = nse.calculate_terms_Jacobian();

   assert_true(((terms_Jacobian.calculate_transpose()).dot(evaluation_terms)*2.0 - objective_gradient).calculate_absolute_value() < 1.0e-3, LOG);

}

void UnscalingLayerTest::test_calculate_derivatives(void)
{
   message += "test_calculate_derivatives\n";

   NumericalDifferentiation nd;

   UnscalingLayer ul;

   ul.set_display(false);

   Vector<double> inputs;
   Vector<double> derivative;
   Vector<double> numerical_derivative;

   // Test

   ul.set(1);

   ul.set_unscaling_method(UnscalingLayer::MinimumMaximum);

   inputs.set(1, 0.0);

   derivative = ul.calculate_derivatives(inputs);

   assert_true(derivative == 1.0, LOG);

   // Test

   ul.set(1);

   ul.set_unscaling_method(UnscalingLayer::MeanStandardDeviation);

   inputs.set(1, 0.0);

   derivative = ul.calculate_derivatives(inputs);

   assert_true(derivative == 1.0, LOG);

   // Test

   if(numerical_differentiation_tests)
   {
      ul.set(3);

      ul.initialize_random();

      ul.set_unscaling_method(UnscalingLayer::MinimumMaximum);

      inputs.set(3);
      inputs.randomize_normal();

      derivative = ul.calculate_derivatives(inputs);
      numerical_derivative = nd.calculate_derivative(ul, &UnscalingLayer::calculate_outputs, inputs);

      assert_true((derivative-numerical_derivative).calculate_absolute_value() < 1.0e-3, LOG);
   }

   // Test

   if(numerical_differentiation_tests)
   {
      ul.set(3);

      ul.initialize_random();

      ul.set_unscaling_method(UnscalingLayer::MeanStandardDeviation);

      inputs.set(3);
      inputs.randomize_normal();

      derivative = ul.calculate_derivatives(inputs);
      numerical_derivative = nd.calculate_derivative(ul, &UnscalingLayer::calculate_outputs, inputs);

      assert_true((derivative-numerical_derivative).calculate_absolute_value() < 1.0e-3, LOG);
   }
}

void MinkowskiErrorTest::test_calculate_gradient(void)
{
   message += "test_calculate_gradient\n";

   NumericalDifferentiation nd;

   NeuralNetwork nn;
   Vector<size_t> architecture;

   Vector<double> parameters;

   DataSet ds;

   MinkowskiError me(&nn, &ds);

   Vector<double> gradient;
   Vector<double> numerical_gradient;

   // Test

   nn.set(1,1,1);

   nn.initialize_parameters(0.0);

   ds.set(1,1,1);

   ds.initialize_data(0.0);

   gradient = me.calculate_gradient();

   assert_true(gradient.size() == nn.count_parameters_number(), LOG);
   assert_true(gradient == 0.0, LOG);

   // Test 

   nn.set(3,4,2);
   nn.initialize_parameters(0.0);

   ds.set(3, 2, 5);
   me.set(&nn, &ds);
   ds.initialize_data(0.0);

   gradient = me.calculate_gradient();

   assert_true(gradient.size() == nn.count_parameters_number(), LOG);
   assert_true(gradient == 0.0, LOG);

   // Test

   architecture.set(3);
   architecture[0] = 2;
   architecture[1] = 1;
   architecture[2] = 3;

   nn.set(architecture);
   nn.initialize_parameters(0.0);

   ds.set(2, 3, 5);
   me.set(&nn, &ds);
   ds.initialize_data(0.0);

   gradient = me.calculate_gradient();

   assert_true(gradient.size() == nn.count_parameters_number(), LOG);
   assert_true(gradient == 0.0, LOG);

   // Test

   nn.set(1,1,1);

   nn.initialize_parameters(0.0);

   ds.set(1,1,1);

   ds.initialize_data(0.0);

   gradient = me.calculate_gradient();

   assert_true(gradient.size() == nn.count_parameters_number(), LOG);
   assert_true(gradient == 0.0, LOG);

   // Test 

   nn.set(3,4,2);
   nn.initialize_parameters(0.0);

   ds.set(3,2,5);
   me.set(&nn, &ds);
   ds.initialize_data(0.0);

   gradient = me.calculate_gradient();

   assert_true(gradient.size() == nn.count_parameters_number(), LOG);
   assert_true(gradient == 0.0, LOG);

   // Test

   architecture.set(3);
   architecture[0] = 2;
   architecture[1] = 1;
//.........这里部分代码省略.........