Java Activation.SIGMOID属性代码示例

@Test
public void testPerOutputMaskingMLP() {
    int nIn = 6;
    int layerSize = 4;

    INDArray mask1 = Nd4j.create(new double[] {1, 0, 0, 1, 0});
    INDArray mask3 = Nd4j.create(new double[][] {{1, 1, 1, 1, 1}, {0, 1, 0, 1, 0}, {1, 0, 0, 1, 1}});
    INDArray[] labelMasks = new INDArray[] {mask1, mask3};

    ILossFunction[] lossFunctions = new ILossFunction[] {new LossBinaryXENT(),
                    //                new LossCosineProximity(),    //Doesn't support per-output masking, as it doesn't make sense for cosine proximity
                    new LossHinge(), new LossKLD(), new LossKLD(), new LossL1(), new LossL2(), new LossMAE(),
                    new LossMAE(), new LossMAPE(), new LossMAPE(),
                    //                new LossMCXENT(),             //Per output masking on MCXENT+Softmax: not yet supported
                    new LossMCXENT(), new LossMSE(), new LossMSE(), new LossMSLE(), new LossMSLE(),
                    new LossNegativeLogLikelihood(), new LossPoisson(), new LossSquaredHinge()};

    Activation[] act = new Activation[] {Activation.SIGMOID, //XENT
                    //                Activation.TANH,
                    Activation.TANH, //Hinge
                    Activation.SIGMOID, //KLD
                    Activation.SOFTMAX, //KLD + softmax
                    Activation.TANH, //L1
                    Activation.TANH, //L2
                    Activation.TANH, //MAE
                    Activation.SOFTMAX, //MAE + softmax
                    Activation.TANH, //MAPE
                    Activation.SOFTMAX, //MAPE + softmax
                    //                Activation.SOFTMAX, //MCXENT + softmax: see comment above
                    Activation.SIGMOID, //MCXENT + sigmoid
                    Activation.TANH, //MSE
                    Activation.SOFTMAX, //MSE + softmax
                    Activation.SIGMOID, //MSLE - needs positive labels/activations (due to log)
                    Activation.SOFTMAX, //MSLE + softmax
                    Activation.SIGMOID, //NLL
                    Activation.SIGMOID, //Poisson
                    Activation.TANH //Squared hinge
    };

    for (INDArray labelMask : labelMasks) {

        int minibatch = labelMask.size(0);
        int nOut = labelMask.size(1);

        for (int i = 0; i < lossFunctions.length; i++) {
            ILossFunction lf = lossFunctions[i];
            Activation a = act[i];


            MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().updater(new NoOp())
                            .weightInit(WeightInit.DISTRIBUTION).dist(new NormalDistribution(0, 1)).seed(12345)
                            .list()
                            .layer(0, new DenseLayer.Builder().nIn(nIn).nOut(layerSize).activation(Activation.TANH)
                                            .build())
                            .layer(1, new OutputLayer.Builder().nIn(layerSize).nOut(nOut).lossFunction(lf)
                                            .activation(a).build())
                            .build();

            MultiLayerNetwork net = new MultiLayerNetwork(conf);
            net.init();

            INDArray[] fl = LossFunctionGradientCheck.getFeaturesAndLabels(lf, minibatch, nIn, nOut, 12345);
            INDArray features = fl[0];
            INDArray labels = fl[1];

            String msg = "testPerOutputMaskingMLP(): maskShape = " + Arrays.toString(labelMask.shape())
                            + ", loss function = " + lf + ", activation = " + a;

            System.out.println(msg);

            boolean gradOK = GradientCheckUtil.checkGradients(net, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR,
                            DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, features, labels, null, labelMask);

            assertTrue(msg, gradOK);
        }
    }
}
 

@Test
public void testGradientCNNMLN() {
    //Parameterized test, testing combinations of:
    // (a) activation function
    // (b) Whether to test at random initialization, or after some learning (i.e., 'characteristic mode of operation')
    // (c) Loss function (with specified output activations)
    Activation[] activFns = {Activation.SIGMOID, Activation.TANH};
    boolean[] characteristic = {false, true}; //If true: run some backprop steps first

    LossFunctions.LossFunction[] lossFunctions =
            {LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD, LossFunctions.LossFunction.MSE};
    Activation[] outputActivations = {Activation.SOFTMAX, Activation.TANH}; //i.e., lossFunctions[i] used with outputActivations[i] here

    DataSet ds = new IrisDataSetIterator(150, 150).next();
    ds.normalizeZeroMeanZeroUnitVariance();
    INDArray input = ds.getFeatureMatrix();
    INDArray labels = ds.getLabels();

    for (Activation afn : activFns) {
        for (boolean doLearningFirst : characteristic) {
            for (int i = 0; i < lossFunctions.length; i++) {
                LossFunctions.LossFunction lf = lossFunctions[i];
                Activation outputActivation = outputActivations[i];

                MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder()
                        .optimizationAlgo(OptimizationAlgorithm.CONJUGATE_GRADIENT).updater(new NoOp())
                        .weightInit(WeightInit.XAVIER).seed(12345L).list()
                        .layer(0, new ConvolutionLayer.Builder(1, 1).nOut(6).activation(afn).build())
                        .layer(1, new OutputLayer.Builder(lf).activation(outputActivation).nOut(3).build())
                        .setInputType(InputType.convolutionalFlat(1, 4, 1)).pretrain(false).backprop(true);

                MultiLayerConfiguration conf = builder.build();

                MultiLayerNetwork mln = new MultiLayerNetwork(conf);
                mln.init();
                String name = new Object() {
                }.getClass().getEnclosingMethod().getName();

                if (doLearningFirst) {
                    //Run a number of iterations of learning
                    mln.setInput(ds.getFeatures());
                    mln.setLabels(ds.getLabels());
                    mln.computeGradientAndScore();
                    double scoreBefore = mln.score();
                    for (int j = 0; j < 10; j++)
                        mln.fit(ds);
                    mln.computeGradientAndScore();
                    double scoreAfter = mln.score();
                    //Can't test in 'characteristic mode of operation' if not learning
                    String msg = name + " - score did not (sufficiently) decrease during learning - activationFn="
                            + afn + ", lossFn=" + lf + ", outputActivation=" + outputActivation
                            + ", doLearningFirst= " + doLearningFirst + " (before=" + scoreBefore
                            + ", scoreAfter=" + scoreAfter + ")";
                    assertTrue(msg, scoreAfter < 0.8 * scoreBefore);
                }

                if (PRINT_RESULTS) {
                    System.out.println(name + " - activationFn=" + afn + ", lossFn=" + lf + ", outputActivation="
                            + outputActivation + ", doLearningFirst=" + doLearningFirst);
                    for (int j = 0; j < mln.getnLayers(); j++)
                        System.out.println("Layer " + j + " # params: " + mln.getLayer(j).numParams());
                }

                boolean gradOK = GradientCheckUtil.checkGradients(mln, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR,
                        DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels);

                assertTrue(gradOK);
            }
        }
    }
}
 

@Test
public void testCnnWithSubsampling() {
    Nd4j.getRandom().setSeed(12345);
    int nOut = 4;

    int[] minibatchSizes = {1, 3};
    int width = 5;
    int height = 5;
    int inputDepth = 1;

    int[] kernel = {2, 2};
    int[] stride = {1, 1};
    int[] padding = {0, 0};
    int pnorm = 2;

    Activation[] activations = {Activation.SIGMOID, Activation.TANH};
    SubsamplingLayer.PoolingType[] poolingTypes =
            new SubsamplingLayer.PoolingType[]{SubsamplingLayer.PoolingType.MAX,
                    SubsamplingLayer.PoolingType.AVG, SubsamplingLayer.PoolingType.PNORM};

    for (Activation afn : activations) {
        for (SubsamplingLayer.PoolingType poolingType : poolingTypes) {
            for (int minibatchSize : minibatchSizes) {
                INDArray input = Nd4j.rand(minibatchSize, width * height * inputDepth);
                INDArray labels = Nd4j.zeros(minibatchSize, nOut);
                for (int i = 0; i < minibatchSize; i++) {
                    labels.putScalar(new int[]{i, i % nOut}, 1.0);
                }

                MultiLayerConfiguration conf =
                        new NeuralNetConfiguration.Builder().updater(new NoOp())
                                .weightInit(WeightInit.DISTRIBUTION)
                                .dist(new NormalDistribution(0, 1))
                                .list().layer(0,
                                new ConvolutionLayer.Builder(kernel,
                                        stride, padding).nIn(inputDepth)
                                        .nOut(3).build())//output: (5-2+0)/1+1 = 4
                                .layer(1, new SubsamplingLayer.Builder(poolingType)
                                        .kernelSize(kernel).stride(stride).padding(padding)
                                        .pnorm(pnorm).build()) //output: (4-2+0)/1+1 =3 -> 3x3x3
                                .layer(2, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
                                        .activation(Activation.SOFTMAX).nIn(3 * 3 * 3)
                                        .nOut(4).build())
                                .setInputType(InputType.convolutionalFlat(height, width,
                                        inputDepth))
                                .build();

                MultiLayerNetwork net = new MultiLayerNetwork(conf);
                net.init();

                String msg = "PoolingType=" + poolingType + ", minibatch=" + minibatchSize + ", activationFn="
                        + afn;

                if (PRINT_RESULTS) {
                    System.out.println(msg);
                    for (int j = 0; j < net.getnLayers(); j++)
                        System.out.println("Layer " + j + " # params: " + net.getLayer(j).numParams());
                }

                boolean gradOK = GradientCheckUtil.checkGradients(net, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR,
                        DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels);

                assertTrue(msg, gradOK);
            }
        }
    }
}
 

@Test
public void testCnnWithSubsamplingV2() {
    Nd4j.getRandom().setSeed(12345);
    int nOut = 4;

    int[] minibatchSizes = {1, 3};
    int width = 5;
    int height = 5;
    int inputDepth = 1;

    int[] kernel = {2, 2};
    int[] stride = {1, 1};
    int[] padding = {0, 0};
    int pNorm = 3;

    Activation[] activations = {Activation.SIGMOID, Activation.TANH};
    SubsamplingLayer.PoolingType[] poolingTypes =
            new SubsamplingLayer.PoolingType[]{SubsamplingLayer.PoolingType.MAX,
                    SubsamplingLayer.PoolingType.AVG, SubsamplingLayer.PoolingType.PNORM};

    for (Activation afn : activations) {
        for (SubsamplingLayer.PoolingType poolingType : poolingTypes) {
            for (int minibatchSize : minibatchSizes) {
                INDArray input = Nd4j.rand(minibatchSize, width * height * inputDepth);
                INDArray labels = Nd4j.zeros(minibatchSize, nOut);
                for (int i = 0; i < minibatchSize; i++) {
                    labels.putScalar(new int[]{i, i % nOut}, 1.0);
                }

                MultiLayerConfiguration conf =
                        new NeuralNetConfiguration.Builder().updater(new NoOp()).weightInit(WeightInit.DISTRIBUTION)
                                .dist(new NormalDistribution(0, 1))
                                .list().layer(0,
                                new ConvolutionLayer.Builder(kernel,
                                        stride, padding).nIn(inputDepth)
                                        .nOut(3).build())//output: (5-2+0)/1+1 = 4
                                .layer(1, new SubsamplingLayer.Builder(poolingType)
                                        .kernelSize(kernel).stride(stride).padding(padding)
                                        .pnorm(pNorm).build()) //output: (4-2+0)/1+1 =3 -> 3x3x3
                                .layer(2, new ConvolutionLayer.Builder(kernel, stride, padding)
                                        .nIn(3).nOut(2).build()) //Output: (3-2+0)/1+1 = 2
                                .layer(3, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
                                        .activation(Activation.SOFTMAX).nIn(2 * 2 * 2)
                                        .nOut(4).build())
                                .setInputType(InputType.convolutionalFlat(height, width,
                                        inputDepth))
                                .build();

                MultiLayerNetwork net = new MultiLayerNetwork(conf);
                net.init();

                String msg = "PoolingType=" + poolingType + ", minibatch=" + minibatchSize + ", activationFn="
                        + afn;
                System.out.println(msg);

                boolean gradOK = GradientCheckUtil.checkGradients(net, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR,
                        DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels);

                assertTrue(msg, gradOK);
            }
        }
    }
}
 

@Test
public void testCnnMultiLayer() {
    int nOut = 2;

    int[] minibatchSizes = {1, 2, 5};
    int width = 5;
    int height = 5;
    int[] inputDepths = {1, 2, 4};

    Activation[] activations = {Activation.SIGMOID, Activation.TANH};
    SubsamplingLayer.PoolingType[] poolingTypes = new SubsamplingLayer.PoolingType[]{
            SubsamplingLayer.PoolingType.MAX, SubsamplingLayer.PoolingType.AVG};

    Nd4j.getRandom().setSeed(12345);

    for (int inputDepth : inputDepths) {
        for (Activation afn : activations) {
            for (SubsamplingLayer.PoolingType poolingType : poolingTypes) {
                for (int minibatchSize : minibatchSizes) {
                    INDArray input = Nd4j.rand(minibatchSize, width * height * inputDepth);
                    INDArray labels = Nd4j.zeros(minibatchSize, nOut);
                    for (int i = 0; i < minibatchSize; i++) {
                        labels.putScalar(new int[]{i, i % nOut}, 1.0);
                    }

                    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).updater(new NoOp())
                            .activation(afn)
                            .list()
                            .layer(0, new ConvolutionLayer.Builder().kernelSize(2, 2).stride(1, 1)
                                    .padding(0, 0).nIn(inputDepth).nOut(2).build())//output: (5-2+0)/1+1 = 4
                            .layer(1, new ConvolutionLayer.Builder().nIn(2).nOut(2).kernelSize(2, 2)
                                    .stride(1, 1).padding(0, 0).build()) //(4-2+0)/1+1 = 3
                            .layer(2, new ConvolutionLayer.Builder().nIn(2).nOut(2).kernelSize(2, 2)
                                    .stride(1, 1).padding(0, 0).build()) //(3-2+0)/1+1 = 2
                            .layer(3, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
                                    .activation(Activation.SOFTMAX).nIn(2 * 2 * 2).nOut(nOut)
                                    .build())
                            .setInputType(InputType.convolutionalFlat(height, width, inputDepth)).build();

                    assertEquals(ConvolutionMode.Truncate,
                            ((ConvolutionLayer) conf.getConf(0).getLayer()).getConvolutionMode());

                    MultiLayerNetwork net = new MultiLayerNetwork(conf);
                    net.init();

                    for (int i = 0; i < 4; i++) {
                        System.out.println("nParams, layer " + i + ": " + net.getLayer(i).numParams());
                    }

                    String msg = "PoolingType=" + poolingType + ", minibatch=" + minibatchSize + ", activationFn="
                            + afn;
                    System.out.println(msg);

                    boolean gradOK = GradientCheckUtil.checkGradients(net, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR,
                            DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels);

                    assertTrue(msg, gradOK);
                }
            }
        }
    }
}
 

@Test
public void testDeconvolution2D() {
    int nOut = 2;

    int[] minibatchSizes = new int[]{1, 3, 1, 3, 1, 3, 1, 3};
    int[] kernelSizes = new int[]{1, 1, 3, 3, 1, 1, 3, 3};
    int[] strides = {1, 1, 1, 1, 2, 2, 2, 2};
    int[] dilation = {1, 2, 2, 1, 1, 1, 2, 2};
    Activation[] activations = new Activation[]{Activation.SIGMOID, Activation.TANH, Activation.TANH, Activation.TANH, Activation.TANH,  Activation.SIGMOID, Activation.SIGMOID, Activation.SIGMOID};
    ConvolutionMode[] cModes = new ConvolutionMode[]{Truncate, Truncate, Truncate, Truncate, Truncate, Truncate, Truncate, Truncate};
    int width = 7;
    int height = 7;
    int inputDepth = 3;

    Nd4j.getRandom().setSeed(12345);

    for (int i = 0; i < minibatchSizes.length; i++) {
        int minibatchSize = minibatchSizes[i];
        int k = kernelSizes[i];
        int s = strides[i];
        int d = dilation[i];
        ConvolutionMode cm = cModes[i];
        Activation act = activations[i];


        int w = d * width;
        int h = d * height;

        INDArray input = Nd4j.rand(minibatchSize, w * h * inputDepth);
        INDArray labels = Nd4j.zeros(minibatchSize, nOut);
        for (int j = 0; j < minibatchSize; j++) {
            labels.putScalar(new int[]{j, j % nOut}, 1.0);
        }

        NeuralNetConfiguration.ListBuilder b = new NeuralNetConfiguration.Builder().seed(12345)
                .updater(new NoOp())
                .activation(act)
                .list()
                .layer(new Deconvolution2D.Builder().name("deconvolution_2D_layer")
                        .kernelSize(k, k)
                        .stride(s, s)
                        .dilation(d, d)
                        .convolutionMode(cm)
                        .nIn(inputDepth).nOut(nOut).build());

        MultiLayerConfiguration conf = b.layer(new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
                .activation(Activation.SOFTMAX).nOut(nOut).build())
                .setInputType(InputType.convolutionalFlat(h, w, inputDepth)).build();

        MultiLayerNetwork net = new MultiLayerNetwork(conf);
        net.init();

        for (int j = 0; j < net.getLayers().length; j++) {
            System.out.println("nParams, layer " + j + ": " + net.getLayer(j).numParams());
        }

        String msg = " - mb=" + minibatchSize + ", k="
                + k + ", s=" + s + ", d=" + d + ", cm=" + cm;
        System.out.println(msg);

        boolean gradOK = GradientCheckUtil.checkGradients(net, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR,
                DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels);

        assertTrue(msg, gradOK);
    }
}
 

@Test
public void testCnn1DWithZeroPadding1D() {
    Nd4j.getRandom().setSeed(1337);

    int[] minibatchSizes = {1, 3};
    int length = 7;
    int convNIn = 2;
    int convNOut1 = 3;
    int convNOut2 = 4;
    int finalNOut = 4;


    int[] kernels = {1, 2, 4};
    int stride = 1;
    int pnorm = 2;

    int padding = 0;
    int zeroPadding = 2;
    int paddedLength = length + 2 * zeroPadding;

    Activation[] activations = {Activation.SIGMOID};
    SubsamplingLayer.PoolingType[] poolingTypes =
            new SubsamplingLayer.PoolingType[] {SubsamplingLayer.PoolingType.MAX,
                    SubsamplingLayer.PoolingType.AVG, SubsamplingLayer.PoolingType.PNORM};

    for (Activation afn : activations) {
        for (SubsamplingLayer.PoolingType poolingType : poolingTypes) {
            for (int minibatchSize : minibatchSizes) {
                for (int kernel : kernels) {
                    INDArray input = Nd4j.rand(new int[] {minibatchSize, convNIn, length});
                    INDArray labels = Nd4j.zeros(minibatchSize, finalNOut, paddedLength);
                    for (int i = 0; i < minibatchSize; i++) {
                        for (int j = 0; j < paddedLength; j++) {
                            labels.putScalar(new int[] {i, i % finalNOut, j}, 1.0);
                        }
                    }

                    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
                            .updater(new NoOp()).weightInit(WeightInit.DISTRIBUTION)
                            .dist(new NormalDistribution(0, 1)).convolutionMode(ConvolutionMode.Same).list()
                            .layer(new Convolution1DLayer.Builder().activation(afn).kernelSize(kernel)
                                    .stride(stride).padding(padding).nIn(convNIn).nOut(convNOut1)
                                    .build())
                            .layer(new ZeroPadding1DLayer.Builder(zeroPadding).build())
                            .layer(new Convolution1DLayer.Builder().activation(afn).kernelSize(kernel)
                                    .stride(stride).padding(padding).nIn(convNOut1).nOut(convNOut2)
                                    .build())
                            .layer(new ZeroPadding1DLayer.Builder(0).build())
                            .layer(new Subsampling1DLayer.Builder(poolingType).kernelSize(kernel)
                                    .stride(stride).padding(padding).pnorm(pnorm).build())
                            .layer(new RnnOutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
                                    .activation(Activation.SOFTMAX).nOut(finalNOut).build())
                            .setInputType(InputType.recurrent(convNIn)).build();

                    String json = conf.toJson();
                    MultiLayerConfiguration c2 = MultiLayerConfiguration.fromJson(json);
                    assertEquals(conf, c2);

                    MultiLayerNetwork net = new MultiLayerNetwork(conf);
                    net.init();

                    String msg = "PoolingType=" + poolingType + ", minibatch=" + minibatchSize + ", activationFn="
                            + afn + ", kernel = " + kernel;

                    if (PRINT_RESULTS) {
                        System.out.println(msg);
                        for (int j = 0; j < net.getnLayers(); j++)
                            System.out.println("Layer " + j + " # params: " + net.getLayer(j).numParams());
                    }

                    boolean gradOK = GradientCheckUtil.checkGradients(net, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR,
                            DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels);

                    assertTrue(msg, gradOK);
                }
            }
        }
    }
}
 

@Test
public void testCnn1DWithSubsampling1D() {
    Nd4j.getRandom().setSeed(12345);

    int[] minibatchSizes = {1, 3};
    int length = 7;
    int convNIn = 2;
    int convNOut1 = 3;
    int convNOut2 = 4;
    int finalNOut = 4;

    int[] kernels = {1, 2, 4};
    int stride = 1;
    int padding = 0;
    int pnorm = 2;

    Activation[] activations = {Activation.SIGMOID, Activation.TANH};
    SubsamplingLayer.PoolingType[] poolingTypes =
                    new SubsamplingLayer.PoolingType[] {SubsamplingLayer.PoolingType.MAX,
                                    SubsamplingLayer.PoolingType.AVG, SubsamplingLayer.PoolingType.PNORM};

    for (Activation afn : activations) {
        for (SubsamplingLayer.PoolingType poolingType : poolingTypes) {
            for (int minibatchSize : minibatchSizes) {
                for (int kernel : kernels) {
                    INDArray input = Nd4j.rand(new int[] {minibatchSize, convNIn, length});
                    INDArray labels = Nd4j.zeros(minibatchSize, finalNOut, length);
                    for (int i = 0; i < minibatchSize; i++) {
                        for (int j = 0; j < length; j++) {
                            labels.putScalar(new int[] {i, i % finalNOut, j}, 1.0);
                        }
                    }

                    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
                                    .updater(new NoOp()).weightInit(WeightInit.DISTRIBUTION)
                                    .dist(new NormalDistribution(0, 1)).convolutionMode(ConvolutionMode.Same).list()
                                    .layer(0, new Convolution1DLayer.Builder().activation(afn).kernelSize(kernel)
                                                    .stride(stride).padding(padding).nIn(convNIn).nOut(convNOut1)
                                                    .build())
                                    .layer(1, new Convolution1DLayer.Builder().activation(afn).kernelSize(kernel)
                                                    .stride(stride).padding(padding).nIn(convNOut1).nOut(convNOut2)
                                                    .build())
                                    .layer(2, new Subsampling1DLayer.Builder(poolingType).kernelSize(kernel)
                                                    .stride(stride).padding(padding).pnorm(pnorm).build())
                                    .layer(3, new RnnOutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
                                                    .activation(Activation.SOFTMAX).nOut(finalNOut).build())
                                    .setInputType(InputType.recurrent(convNIn)).build();

                    String json = conf.toJson();
                    MultiLayerConfiguration c2 = MultiLayerConfiguration.fromJson(json);
                    assertEquals(conf, c2);

                    MultiLayerNetwork net = new MultiLayerNetwork(conf);
                    net.init();

                    String msg = "PoolingType=" + poolingType + ", minibatch=" + minibatchSize + ", activationFn="
                                    + afn + ", kernel = " + kernel;

                    if (PRINT_RESULTS) {
                        System.out.println(msg);
                        for (int j = 0; j < net.getnLayers(); j++)
                            System.out.println("Layer " + j + " # params: " + net.getLayer(j).numParams());
                    }

                    boolean gradOK = GradientCheckUtil.checkGradients(net, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR,
                                    DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels);

                    assertTrue(msg, gradOK);
                }
            }
        }
    }
}
 

@Test
public void testBernoulliLogProb() {
    Nd4j.getRandom().setSeed(12345);

    int inputSize = 4;
    int[] mbs = new int[] {1, 2, 5};

    Random r = new Random(12345);

    for (boolean average : new boolean[] {true, false}) {
        for (int minibatch : mbs) {

            INDArray x = Nd4j.zeros(minibatch, inputSize);
            for (int i = 0; i < minibatch; i++) {
                for (int j = 0; j < inputSize; j++) {
                    x.putScalar(i, j, r.nextInt(2));
                }
            }

            INDArray distributionParams = Nd4j.rand(minibatch, inputSize).muli(2).subi(1); //i.e., pre-sigmoid prob
            INDArray prob = Transforms.sigmoid(distributionParams, true);

            ReconstructionDistribution dist = new BernoulliReconstructionDistribution(Activation.SIGMOID);

            double negLogProb = dist.negLogProbability(x, distributionParams, average);

            INDArray exampleNegLogProb = dist.exampleNegLogProbability(x, distributionParams);
            assertArrayEquals(new int[] {minibatch, 1}, exampleNegLogProb.shape());

            //Calculate the same thing, but using Apache Commons math

            double logProbSum = 0.0;
            for (int i = 0; i < minibatch; i++) {
                double exampleSum = 0.0;
                for (int j = 0; j < inputSize; j++) {
                    double p = prob.getDouble(i, j);

                    BinomialDistribution binomial = new BinomialDistribution(1, p); //Bernoulli is a special case of binomial

                    double xVal = x.getDouble(i, j);
                    double thisLogProb = binomial.logProbability((int) xVal);
                    logProbSum += thisLogProb;
                    exampleSum += thisLogProb;
                }
                assertEquals(-exampleNegLogProb.getDouble(i), exampleSum, 1e-6);
            }

            double expNegLogProb;
            if (average) {
                expNegLogProb = -logProbSum / minibatch;
            } else {
                expNegLogProb = -logProbSum;
            }

            //                System.out.println(x);

            //                System.out.println(expNegLogProb + "\t" + logProb + "\t" + (logProb / expNegLogProb));
            assertEquals(expNegLogProb, negLogProb, 1e-6);

            //Also: check random sampling...
            int count = minibatch * inputSize;
            INDArray arr = Nd4j.linspace(-3, 3, count).reshape(minibatch, inputSize);
            INDArray sampleMean = dist.generateAtMean(arr);
            INDArray sampleRandom = dist.generateRandom(arr);

            for (int i = 0; i < minibatch; i++) {
                for (int j = 0; j < inputSize; j++) {
                    double d1 = sampleMean.getDouble(i, j);
                    double d2 = sampleRandom.getDouble(i, j);
                    assertTrue(d1 >= 0.0 || d1 <= 1.0); //Mean value - probability... could do 0 or 1 (based on most likely) but that isn't very useful...
                    assertTrue(d2 == 0.0 || d2 == 1.0);
                }
            }
        }
    }
}