Java Activation.TANH属性代码示例

@Test
    public void elementWiseMultiplicationLayerTest(){

        for(Activation a : new Activation[]{Activation.IDENTITY, Activation.TANH}) {

            ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder()
                    .optimizationAlgo(OptimizationAlgorithm.CONJUGATE_GRADIENT).updater(new NoOp())
                    .seed(12345L)
                    .weightInit(new UniformDistribution(0, 1))
                    .graphBuilder()
                    .addInputs("features")
                    .addLayer("dense", new DenseLayer.Builder().nIn(4).nOut(4)
                            .activation(Activation.TANH)
                            .build(), "features")
                    .addLayer("elementWiseMul", new ElementWiseMultiplicationLayer.Builder().nIn(4).nOut(4)
                            .activation(a)
                            .build(), "dense")
                    .addLayer("loss", new LossLayer.Builder(LossFunctions.LossFunction.COSINE_PROXIMITY)
                            .activation(Activation.IDENTITY).build(), "elementWiseMul")
                    .setOutputs("loss")
                    .pretrain(false).backprop(true).build();

            ComputationGraph netGraph = new ComputationGraph(conf);
            netGraph.init();

            log.info("params before learning: " + netGraph.getLayer(1).paramTable());

            //Run a number of iterations of learning manually make some pseudo data
            //the ides is simple: since we do a element wise multiplication layer (just a scaling), we want the cos sim
            // is mainly decided by the fourth value, if everything runs well, we will get a large weight for the fourth value

            INDArray features = Nd4j.create(new double[][]{{1, 2, 3, 4}, {1, 2, 3, 1}, {1, 2, 3, 0}});
            INDArray labels = Nd4j.create(new double[][]{{1, 1, 1, 8}, {1, 1, 1, 2}, {1, 1, 1, 1}});

            netGraph.setInputs(features);
            netGraph.setLabels(labels);
            netGraph.computeGradientAndScore();
            double scoreBefore = netGraph.score();

            String msg;
            for (int epoch = 0; epoch < 5; epoch++)
                netGraph.fit(new INDArray[]{features}, new INDArray[]{labels});
            netGraph.computeGradientAndScore();
            double scoreAfter = netGraph.score();
            //Can't test in 'characteristic mode of operation' if not learning
            msg = "elementWiseMultiplicationLayerTest() - score did not (sufficiently) decrease during learning - activationFn="
                    + "Id" + ", lossFn=" + "Cos-sim" + ", outputActivation=" + "Id"
                    + ", doLearningFirst=" + "true" + " (before=" + scoreBefore
                    + ", scoreAfter=" + scoreAfter + ")";
            assertTrue(msg, scoreAfter < 0.8 * scoreBefore);

//        expectation in case linear regression(with only element wise multiplication layer): large weight for the fourth weight
            log.info("params after learning: " + netGraph.getLayer(1).paramTable());

            boolean gradOK = checkGradients(netGraph, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR,
                    DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, new INDArray[]{features}, new INDArray[]{labels});

            msg = "elementWiseMultiplicationLayerTest() - activationFn=" + "ID" + ", lossFn=" + "Cos-sim"
                    + ", outputActivation=" + "Id" + ", doLearningFirst=" + "true";
            assertTrue(msg, gradOK);

            TestUtils.testModelSerialization(netGraph);
        }
    }
 

@Test
public void gradientCheckMaskingOutputSimple() {

    int timeSeriesLength = 5;
    boolean[][] mask = new boolean[5][0];
    mask[0] = new boolean[] {true, true, true, true, true}; //No masking
    mask[1] = new boolean[] {false, true, true, true, true}; //mask first output time step
    mask[2] = new boolean[] {false, false, false, false, true}; //time series classification: mask all but last
    mask[3] = new boolean[] {false, false, true, false, true}; //time series classification w/ variable length TS
    mask[4] = new boolean[] {true, true, true, false, true}; //variable length TS

    int nIn = 4;
    int layerSize = 3;

    GradientCheckSimpleScenario[] scenarios = new GradientCheckSimpleScenario[] {
                    new GradientCheckSimpleScenario(LossFunctions.LossFunction.MCXENT.getILossFunction(),
                                    Activation.SOFTMAX, 2, 2),
                    new GradientCheckSimpleScenario(LossMixtureDensity.builder().gaussians(2).labelWidth(3).build(),
                                    Activation.TANH, 10, 3),
                    new GradientCheckSimpleScenario(LossMixtureDensity.builder().gaussians(2).labelWidth(4).build(),
                                    Activation.IDENTITY, 12, 4),
                    new GradientCheckSimpleScenario(LossFunctions.LossFunction.L2.getILossFunction(),
                                    Activation.SOFTMAX, 2, 2)};

    for (GradientCheckSimpleScenario s : scenarios) {

        Random r = new Random(12345L);
        INDArray input = Nd4j.zeros(1, nIn, timeSeriesLength);
        for (int m = 0; m < 1; m++) {
            for (int j = 0; j < nIn; j++) {
                for (int k = 0; k < timeSeriesLength; k++) {
                    input.putScalar(new int[] {m, j, k}, r.nextDouble() - 0.5);
                }
            }
        }

        INDArray labels = Nd4j.zeros(1, s.labelWidth, timeSeriesLength);
        for (int m = 0; m < 1; m++) {
            for (int j = 0; j < timeSeriesLength; j++) {
                int idx = r.nextInt(s.labelWidth);
                labels.putScalar(new int[] {m, idx, j}, 1.0f);
            }
        }

        for (int i = 0; i < mask.length; i++) {

            //Create mask array:
            INDArray maskArr = Nd4j.create(1, timeSeriesLength);
            for (int j = 0; j < mask[i].length; j++) {
                maskArr.putScalar(new int[] {0, j}, mask[i][j] ? 1.0 : 0.0);
            }

            MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345L)
                            .list()
                            .layer(0, new GravesLSTM.Builder().nIn(nIn).nOut(layerSize)
                                            .weightInit(WeightInit.DISTRIBUTION).dist(new NormalDistribution(0, 1))
                                            .updater(new NoOp()).build())
                            .layer(1, new RnnOutputLayer.Builder(s.lf).activation(s.act).nIn(layerSize).nOut(s.nOut)
                                            .weightInit(WeightInit.DISTRIBUTION).dist(new NormalDistribution(0, 1))
                                            .updater(new NoOp()).build())
                            .pretrain(false).backprop(true).build();
            MultiLayerNetwork mln = new MultiLayerNetwork(conf);
            mln.init();

            boolean gradOK = GradientCheckUtil.checkGradients(mln, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR,
                            DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels, null, maskArr);

            String msg = "gradientCheckMaskingOutputSimple() - timeSeriesLength=" + timeSeriesLength
                            + ", miniBatchSize=" + 1;
            assertTrue(msg, gradOK);
        }
    }
}
 

@Test
public void testPerOutputMaskingMLP() {
    int nIn = 6;
    int layerSize = 4;

    INDArray mask1 = Nd4j.create(new double[] {1, 0, 0, 1, 0});
    INDArray mask3 = Nd4j.create(new double[][] {{1, 1, 1, 1, 1}, {0, 1, 0, 1, 0}, {1, 0, 0, 1, 1}});
    INDArray[] labelMasks = new INDArray[] {mask1, mask3};

    ILossFunction[] lossFunctions = new ILossFunction[] {new LossBinaryXENT(),
                    //                new LossCosineProximity(),    //Doesn't support per-output masking, as it doesn't make sense for cosine proximity
                    new LossHinge(), new LossKLD(), new LossKLD(), new LossL1(), new LossL2(), new LossMAE(),
                    new LossMAE(), new LossMAPE(), new LossMAPE(),
                    //                new LossMCXENT(),             //Per output masking on MCXENT+Softmax: not yet supported
                    new LossMCXENT(), new LossMSE(), new LossMSE(), new LossMSLE(), new LossMSLE(),
                    new LossNegativeLogLikelihood(), new LossPoisson(), new LossSquaredHinge()};

    Activation[] act = new Activation[] {Activation.SIGMOID, //XENT
                    //                Activation.TANH,
                    Activation.TANH, //Hinge
                    Activation.SIGMOID, //KLD
                    Activation.SOFTMAX, //KLD + softmax
                    Activation.TANH, //L1
                    Activation.TANH, //L2
                    Activation.TANH, //MAE
                    Activation.SOFTMAX, //MAE + softmax
                    Activation.TANH, //MAPE
                    Activation.SOFTMAX, //MAPE + softmax
                    //                Activation.SOFTMAX, //MCXENT + softmax: see comment above
                    Activation.SIGMOID, //MCXENT + sigmoid
                    Activation.TANH, //MSE
                    Activation.SOFTMAX, //MSE + softmax
                    Activation.SIGMOID, //MSLE - needs positive labels/activations (due to log)
                    Activation.SOFTMAX, //MSLE + softmax
                    Activation.SIGMOID, //NLL
                    Activation.SIGMOID, //Poisson
                    Activation.TANH //Squared hinge
    };

    for (INDArray labelMask : labelMasks) {

        int minibatch = labelMask.size(0);
        int nOut = labelMask.size(1);

        for (int i = 0; i < lossFunctions.length; i++) {
            ILossFunction lf = lossFunctions[i];
            Activation a = act[i];


            MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().updater(new NoOp())
                            .weightInit(WeightInit.DISTRIBUTION).dist(new NormalDistribution(0, 1)).seed(12345)
                            .list()
                            .layer(0, new DenseLayer.Builder().nIn(nIn).nOut(layerSize).activation(Activation.TANH)
                                            .build())
                            .layer(1, new OutputLayer.Builder().nIn(layerSize).nOut(nOut).lossFunction(lf)
                                            .activation(a).build())
                            .build();

            MultiLayerNetwork net = new MultiLayerNetwork(conf);
            net.init();

            INDArray[] fl = LossFunctionGradientCheck.getFeaturesAndLabels(lf, minibatch, nIn, nOut, 12345);
            INDArray features = fl[0];
            INDArray labels = fl[1];

            String msg = "testPerOutputMaskingMLP(): maskShape = " + Arrays.toString(labelMask.shape())
                            + ", loss function = " + lf + ", activation = " + a;

            System.out.println(msg);

            boolean gradOK = GradientCheckUtil.checkGradients(net, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR,
                            DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, features, labels, null, labelMask);

            assertTrue(msg, gradOK);
        }
    }
}
 

@Test
public void testGradientCNNMLN() {
    //Parameterized test, testing combinations of:
    // (a) activation function
    // (b) Whether to test at random initialization, or after some learning (i.e., 'characteristic mode of operation')
    // (c) Loss function (with specified output activations)
    Activation[] activFns = {Activation.SIGMOID, Activation.TANH};
    boolean[] characteristic = {false, true}; //If true: run some backprop steps first

    LossFunctions.LossFunction[] lossFunctions =
            {LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD, LossFunctions.LossFunction.MSE};
    Activation[] outputActivations = {Activation.SOFTMAX, Activation.TANH}; //i.e., lossFunctions[i] used with outputActivations[i] here

    DataSet ds = new IrisDataSetIterator(150, 150).next();
    ds.normalizeZeroMeanZeroUnitVariance();
    INDArray input = ds.getFeatureMatrix();
    INDArray labels = ds.getLabels();

    for (Activation afn : activFns) {
        for (boolean doLearningFirst : characteristic) {
            for (int i = 0; i < lossFunctions.length; i++) {
                LossFunctions.LossFunction lf = lossFunctions[i];
                Activation outputActivation = outputActivations[i];

                MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder()
                        .optimizationAlgo(OptimizationAlgorithm.CONJUGATE_GRADIENT).updater(new NoOp())
                        .weightInit(WeightInit.XAVIER).seed(12345L).list()
                        .layer(0, new ConvolutionLayer.Builder(1, 1).nOut(6).activation(afn).build())
                        .layer(1, new OutputLayer.Builder(lf).activation(outputActivation).nOut(3).build())
                        .setInputType(InputType.convolutionalFlat(1, 4, 1)).pretrain(false).backprop(true);

                MultiLayerConfiguration conf = builder.build();

                MultiLayerNetwork mln = new MultiLayerNetwork(conf);
                mln.init();
                String name = new Object() {
                }.getClass().getEnclosingMethod().getName();

                if (doLearningFirst) {
                    //Run a number of iterations of learning
                    mln.setInput(ds.getFeatures());
                    mln.setLabels(ds.getLabels());
                    mln.computeGradientAndScore();
                    double scoreBefore = mln.score();
                    for (int j = 0; j < 10; j++)
                        mln.fit(ds);
                    mln.computeGradientAndScore();
                    double scoreAfter = mln.score();
                    //Can't test in 'characteristic mode of operation' if not learning
                    String msg = name + " - score did not (sufficiently) decrease during learning - activationFn="
                            + afn + ", lossFn=" + lf + ", outputActivation=" + outputActivation
                            + ", doLearningFirst= " + doLearningFirst + " (before=" + scoreBefore
                            + ", scoreAfter=" + scoreAfter + ")";
                    assertTrue(msg, scoreAfter < 0.8 * scoreBefore);
                }

                if (PRINT_RESULTS) {
                    System.out.println(name + " - activationFn=" + afn + ", lossFn=" + lf + ", outputActivation="
                            + outputActivation + ", doLearningFirst=" + doLearningFirst);
                    for (int j = 0; j < mln.getnLayers(); j++)
                        System.out.println("Layer " + j + " # params: " + mln.getLayer(j).numParams());
                }

                boolean gradOK = GradientCheckUtil.checkGradients(mln, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR,
                        DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels);

                assertTrue(gradOK);
            }
        }
    }
}
 

@Test
public void testCnnWithSubsampling() {
    Nd4j.getRandom().setSeed(12345);
    int nOut = 4;

    int[] minibatchSizes = {1, 3};
    int width = 5;
    int height = 5;
    int inputDepth = 1;

    int[] kernel = {2, 2};
    int[] stride = {1, 1};
    int[] padding = {0, 0};
    int pnorm = 2;

    Activation[] activations = {Activation.SIGMOID, Activation.TANH};
    SubsamplingLayer.PoolingType[] poolingTypes =
            new SubsamplingLayer.PoolingType[]{SubsamplingLayer.PoolingType.MAX,
                    SubsamplingLayer.PoolingType.AVG, SubsamplingLayer.PoolingType.PNORM};

    for (Activation afn : activations) {
        for (SubsamplingLayer.PoolingType poolingType : poolingTypes) {
            for (int minibatchSize : minibatchSizes) {
                INDArray input = Nd4j.rand(minibatchSize, width * height * inputDepth);
                INDArray labels = Nd4j.zeros(minibatchSize, nOut);
                for (int i = 0; i < minibatchSize; i++) {
                    labels.putScalar(new int[]{i, i % nOut}, 1.0);
                }

                MultiLayerConfiguration conf =
                        new NeuralNetConfiguration.Builder().updater(new NoOp())
                                .weightInit(WeightInit.DISTRIBUTION)
                                .dist(new NormalDistribution(0, 1))
                                .list().layer(0,
                                new ConvolutionLayer.Builder(kernel,
                                        stride, padding).nIn(inputDepth)
                                        .nOut(3).build())//output: (5-2+0)/1+1 = 4
                                .layer(1, new SubsamplingLayer.Builder(poolingType)
                                        .kernelSize(kernel).stride(stride).padding(padding)
                                        .pnorm(pnorm).build()) //output: (4-2+0)/1+1 =3 -> 3x3x3
                                .layer(2, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
                                        .activation(Activation.SOFTMAX).nIn(3 * 3 * 3)
                                        .nOut(4).build())
                                .setInputType(InputType.convolutionalFlat(height, width,
                                        inputDepth))
                                .build();

                MultiLayerNetwork net = new MultiLayerNetwork(conf);
                net.init();

                String msg = "PoolingType=" + poolingType + ", minibatch=" + minibatchSize + ", activationFn="
                        + afn;

                if (PRINT_RESULTS) {
                    System.out.println(msg);
                    for (int j = 0; j < net.getnLayers(); j++)
                        System.out.println("Layer " + j + " # params: " + net.getLayer(j).numParams());
                }

                boolean gradOK = GradientCheckUtil.checkGradients(net, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR,
                        DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels);

                assertTrue(msg, gradOK);
            }
        }
    }
}
 

@Test
public void testCnnWithSubsamplingV2() {
    Nd4j.getRandom().setSeed(12345);
    int nOut = 4;

    int[] minibatchSizes = {1, 3};
    int width = 5;
    int height = 5;
    int inputDepth = 1;

    int[] kernel = {2, 2};
    int[] stride = {1, 1};
    int[] padding = {0, 0};
    int pNorm = 3;

    Activation[] activations = {Activation.SIGMOID, Activation.TANH};
    SubsamplingLayer.PoolingType[] poolingTypes =
            new SubsamplingLayer.PoolingType[]{SubsamplingLayer.PoolingType.MAX,
                    SubsamplingLayer.PoolingType.AVG, SubsamplingLayer.PoolingType.PNORM};

    for (Activation afn : activations) {
        for (SubsamplingLayer.PoolingType poolingType : poolingTypes) {
            for (int minibatchSize : minibatchSizes) {
                INDArray input = Nd4j.rand(minibatchSize, width * height * inputDepth);
                INDArray labels = Nd4j.zeros(minibatchSize, nOut);
                for (int i = 0; i < minibatchSize; i++) {
                    labels.putScalar(new int[]{i, i % nOut}, 1.0);
                }

                MultiLayerConfiguration conf =
                        new NeuralNetConfiguration.Builder().updater(new NoOp()).weightInit(WeightInit.DISTRIBUTION)
                                .dist(new NormalDistribution(0, 1))
                                .list().layer(0,
                                new ConvolutionLayer.Builder(kernel,
                                        stride, padding).nIn(inputDepth)
                                        .nOut(3).build())//output: (5-2+0)/1+1 = 4
                                .layer(1, new SubsamplingLayer.Builder(poolingType)
                                        .kernelSize(kernel).stride(stride).padding(padding)
                                        .pnorm(pNorm).build()) //output: (4-2+0)/1+1 =3 -> 3x3x3
                                .layer(2, new ConvolutionLayer.Builder(kernel, stride, padding)
                                        .nIn(3).nOut(2).build()) //Output: (3-2+0)/1+1 = 2
                                .layer(3, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
                                        .activation(Activation.SOFTMAX).nIn(2 * 2 * 2)
                                        .nOut(4).build())
                                .setInputType(InputType.convolutionalFlat(height, width,
                                        inputDepth))
                                .build();

                MultiLayerNetwork net = new MultiLayerNetwork(conf);
                net.init();

                String msg = "PoolingType=" + poolingType + ", minibatch=" + minibatchSize + ", activationFn="
                        + afn;
                System.out.println(msg);

                boolean gradOK = GradientCheckUtil.checkGradients(net, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR,
                        DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels);

                assertTrue(msg, gradOK);
            }
        }
    }
}
 

@Test
public void testCnnMultiLayer() {
    int nOut = 2;

    int[] minibatchSizes = {1, 2, 5};
    int width = 5;
    int height = 5;
    int[] inputDepths = {1, 2, 4};

    Activation[] activations = {Activation.SIGMOID, Activation.TANH};
    SubsamplingLayer.PoolingType[] poolingTypes = new SubsamplingLayer.PoolingType[]{
            SubsamplingLayer.PoolingType.MAX, SubsamplingLayer.PoolingType.AVG};

    Nd4j.getRandom().setSeed(12345);

    for (int inputDepth : inputDepths) {
        for (Activation afn : activations) {
            for (SubsamplingLayer.PoolingType poolingType : poolingTypes) {
                for (int minibatchSize : minibatchSizes) {
                    INDArray input = Nd4j.rand(minibatchSize, width * height * inputDepth);
                    INDArray labels = Nd4j.zeros(minibatchSize, nOut);
                    for (int i = 0; i < minibatchSize; i++) {
                        labels.putScalar(new int[]{i, i % nOut}, 1.0);
                    }

                    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).updater(new NoOp())
                            .activation(afn)
                            .list()
                            .layer(0, new ConvolutionLayer.Builder().kernelSize(2, 2).stride(1, 1)
                                    .padding(0, 0).nIn(inputDepth).nOut(2).build())//output: (5-2+0)/1+1 = 4
                            .layer(1, new ConvolutionLayer.Builder().nIn(2).nOut(2).kernelSize(2, 2)
                                    .stride(1, 1).padding(0, 0).build()) //(4-2+0)/1+1 = 3
                            .layer(2, new ConvolutionLayer.Builder().nIn(2).nOut(2).kernelSize(2, 2)
                                    .stride(1, 1).padding(0, 0).build()) //(3-2+0)/1+1 = 2
                            .layer(3, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
                                    .activation(Activation.SOFTMAX).nIn(2 * 2 * 2).nOut(nOut)
                                    .build())
                            .setInputType(InputType.convolutionalFlat(height, width, inputDepth)).build();

                    assertEquals(ConvolutionMode.Truncate,
                            ((ConvolutionLayer) conf.getConf(0).getLayer()).getConvolutionMode());

                    MultiLayerNetwork net = new MultiLayerNetwork(conf);
                    net.init();

                    for (int i = 0; i < 4; i++) {
                        System.out.println("nParams, layer " + i + ": " + net.getLayer(i).numParams());
                    }

                    String msg = "PoolingType=" + poolingType + ", minibatch=" + minibatchSize + ", activationFn="
                            + afn;
                    System.out.println(msg);

                    boolean gradOK = GradientCheckUtil.checkGradients(net, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR,
                            DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels);

                    assertTrue(msg, gradOK);
                }
            }
        }
    }
}
 

@Test
public void testDeconvolution2D() {
    int nOut = 2;

    int[] minibatchSizes = new int[]{1, 3, 1, 3, 1, 3, 1, 3};
    int[] kernelSizes = new int[]{1, 1, 3, 3, 1, 1, 3, 3};
    int[] strides = {1, 1, 1, 1, 2, 2, 2, 2};
    int[] dilation = {1, 2, 2, 1, 1, 1, 2, 2};
    Activation[] activations = new Activation[]{Activation.SIGMOID, Activation.TANH, Activation.TANH, Activation.TANH, Activation.TANH,  Activation.SIGMOID, Activation.SIGMOID, Activation.SIGMOID};
    ConvolutionMode[] cModes = new ConvolutionMode[]{Truncate, Truncate, Truncate, Truncate, Truncate, Truncate, Truncate, Truncate};
    int width = 7;
    int height = 7;
    int inputDepth = 3;

    Nd4j.getRandom().setSeed(12345);

    for (int i = 0; i < minibatchSizes.length; i++) {
        int minibatchSize = minibatchSizes[i];
        int k = kernelSizes[i];
        int s = strides[i];
        int d = dilation[i];
        ConvolutionMode cm = cModes[i];
        Activation act = activations[i];


        int w = d * width;
        int h = d * height;

        INDArray input = Nd4j.rand(minibatchSize, w * h * inputDepth);
        INDArray labels = Nd4j.zeros(minibatchSize, nOut);
        for (int j = 0; j < minibatchSize; j++) {
            labels.putScalar(new int[]{j, j % nOut}, 1.0);
        }

        NeuralNetConfiguration.ListBuilder b = new NeuralNetConfiguration.Builder().seed(12345)
                .updater(new NoOp())
                .activation(act)
                .list()
                .layer(new Deconvolution2D.Builder().name("deconvolution_2D_layer")
                        .kernelSize(k, k)
                        .stride(s, s)
                        .dilation(d, d)
                        .convolutionMode(cm)
                        .nIn(inputDepth).nOut(nOut).build());

        MultiLayerConfiguration conf = b.layer(new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
                .activation(Activation.SOFTMAX).nOut(nOut).build())
                .setInputType(InputType.convolutionalFlat(h, w, inputDepth)).build();

        MultiLayerNetwork net = new MultiLayerNetwork(conf);
        net.init();

        for (int j = 0; j < net.getLayers().length; j++) {
            System.out.println("nParams, layer " + j + ": " + net.getLayer(j).numParams());
        }

        String msg = " - mb=" + minibatchSize + ", k="
                + k + ", s=" + s + ", d=" + d + ", cm=" + cm;
        System.out.println(msg);

        boolean gradOK = GradientCheckUtil.checkGradients(net, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR,
                DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels);

        assertTrue(msg, gradOK);
    }
}
 

@Test
public void testYoloOutputLayer() {
    int depthIn = 2;
    int[] minibatchSizes = {1, 3};
    int[] widths = new int[]{4, 7};
    int[] heights = new int[]{4, 5};
    int c = 3;
    int b = 3;

    int yoloDepth = b * (5 + c);
    Activation a = Activation.TANH;

    Nd4j.getRandom().setSeed(1234567);

    double[] l1 = new double[]{0.0, 0.3};
    double[] l2 = new double[]{0.0, 0.4};

    for( int wh = 0; wh<widths.length; wh++ ) {

        int w = widths[wh];
        int h = heights[wh];

        Nd4j.getRandom().setSeed(12345);
        INDArray bbPrior = Nd4j.rand(b, 2).muliRowVector(Nd4j.create(new double[]{w, h})).addi(0.1);

        for (int mb : minibatchSizes) {
            for (int i = 0; i < l1.length; i++) {

                Nd4j.getRandom().setSeed(12345);

                INDArray input = Nd4j.rand(new int[]{mb, depthIn, h, w});
                INDArray labels = yoloLabels(mb, c, h, w);

                MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345)
                        .updater(new NoOp())
                        .activation(a)
                        .l1(l1[i]).l2(l2[i])
                        .convolutionMode(ConvolutionMode.Same)
                        .list()
                        .layer(new ConvolutionLayer.Builder().kernelSize(2, 2).stride(1, 1)
                                .nIn(depthIn).nOut(yoloDepth).build())//output: (5-2+0)/1+1 = 4
                        .layer(new Yolo2OutputLayer.Builder()
                                .boundingBoxPriors(bbPrior)
                                .build())
                        .build();

                MultiLayerNetwork net = new MultiLayerNetwork(conf);
                net.init();

                String msg = "testYoloOutputLayer() - minibatch = " + mb + ", w=" + w + ", h=" + h + ", l1=" + l1[i] + ", l2=" + l2[i];
                System.out.println(msg);

                boolean gradOK = GradientCheckUtil.checkGradients(net, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR,
                        DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels);

                assertTrue(msg, gradOK);
            }
        }
    }
}
 

@Test
public void testVaeAsMLP() {
    //Post pre-training: a VAE can be used as a MLP, by taking the mean value from p(z|x) as the output
    //This gradient check tests this part

    Activation[] activFns = {Activation.IDENTITY, Activation.TANH, Activation.IDENTITY, Activation.TANH, Activation.IDENTITY, Activation.TANH};

    LossFunction[] lossFunctions = {LossFunction.MCXENT, LossFunction.MCXENT, LossFunction.MSE, LossFunction.MSE, LossFunction.MCXENT, LossFunction.MSE};
    Activation[] outputActivations = {Activation.SOFTMAX, Activation.SOFTMAX, Activation.TANH, Activation.TANH, Activation.SOFTMAX, Activation.TANH};

    //use l2vals[i] with l1vals[i]
    double[] l2vals = {0.4, 0.0, 0.4, 0.4, 0.0, 0.0};
    double[] l1vals = {0.0, 0.0, 0.5, 0.0, 0.0, 0.5};
    double[] biasL2 = {0.0, 0.0, 0.0, 0.2, 0.0, 0.4};
    double[] biasL1 = {0.0, 0.0, 0.6, 0.0, 0.0, 0.0};

    int[][] encoderLayerSizes = new int[][] {{5}, {5}, {5, 6}, {5, 6}, {5}, {5, 6}};
    int[][] decoderLayerSizes = new int[][] {{6}, {7, 8}, {6}, {7, 8}, {6}, {7, 8}};

    int[] minibatches = new int[]{1,5,4,3,1,4};

    Nd4j.getRandom().setSeed(12345);
    for( int i=0; i<activFns.length; i++ ){
        LossFunction lf = lossFunctions[i];
        Activation outputActivation = outputActivations[i];
        double l2 = l2vals[i];
        double l1 = l1vals[i];
        int[] encoderSizes = encoderLayerSizes[i];
        int[] decoderSizes = decoderLayerSizes[i];
        int minibatch = minibatches[i];
        INDArray input = Nd4j.rand(minibatch, 4);
        INDArray labels = Nd4j.create(minibatch, 3);
        for (int j = 0; j < minibatch; j++) {
            labels.putScalar(j, j % 3, 1.0);
        }
        Activation afn = activFns[i];

        MultiLayerConfiguration conf =
                new NeuralNetConfiguration.Builder().l2(l2).l1(l1)
                        .updater(new NoOp())
                        .l2Bias(biasL2[i]).l1Bias(biasL1[i])
                        .updater(new NoOp()).seed(12345L).list()
                        .layer(0, new VariationalAutoencoder.Builder().nIn(4)
                                .nOut(3).encoderLayerSizes(encoderSizes)
                                .decoderLayerSizes(decoderSizes)
                                .weightInit(WeightInit.DISTRIBUTION)
                                .dist(new NormalDistribution(0, 1))
                                .activation(afn)
                                .build())
                        .layer(1, new OutputLayer.Builder(lf)
                                .activation(outputActivation).nIn(3).nOut(3)
                                .weightInit(WeightInit.DISTRIBUTION)
                                .dist(new NormalDistribution(0, 1))
                                .build())
                        .pretrain(false).backprop(true).build();

        MultiLayerNetwork mln = new MultiLayerNetwork(conf);
        mln.init();

        String msg = "testVaeAsMLP() - activationFn=" + afn + ", lossFn=" + lf
                + ", outputActivation=" + outputActivation + ", encLayerSizes = "
                + Arrays.toString(encoderSizes) + ", decLayerSizes = "
                + Arrays.toString(decoderSizes) + ", l2=" + l2 + ", l1=" + l1;
        if (PRINT_RESULTS) {
            System.out.println(msg);
            for (int j = 0; j < mln.getnLayers(); j++)
                System.out.println("Layer " + j + " # params: " + mln.getLayer(j).numParams());
        }

        boolean gradOK = GradientCheckUtil.checkGradients(mln, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR,
                DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input,
                labels);
        assertTrue(msg, gradOK);
    }
}
 

@Test
public void testCnn1DWithSubsampling1D() {
    Nd4j.getRandom().setSeed(12345);

    int[] minibatchSizes = {1, 3};
    int length = 7;
    int convNIn = 2;
    int convNOut1 = 3;
    int convNOut2 = 4;
    int finalNOut = 4;

    int[] kernels = {1, 2, 4};
    int stride = 1;
    int padding = 0;
    int pnorm = 2;

    Activation[] activations = {Activation.SIGMOID, Activation.TANH};
    SubsamplingLayer.PoolingType[] poolingTypes =
                    new SubsamplingLayer.PoolingType[] {SubsamplingLayer.PoolingType.MAX,
                                    SubsamplingLayer.PoolingType.AVG, SubsamplingLayer.PoolingType.PNORM};

    for (Activation afn : activations) {
        for (SubsamplingLayer.PoolingType poolingType : poolingTypes) {
            for (int minibatchSize : minibatchSizes) {
                for (int kernel : kernels) {
                    INDArray input = Nd4j.rand(new int[] {minibatchSize, convNIn, length});
                    INDArray labels = Nd4j.zeros(minibatchSize, finalNOut, length);
                    for (int i = 0; i < minibatchSize; i++) {
                        for (int j = 0; j < length; j++) {
                            labels.putScalar(new int[] {i, i % finalNOut, j}, 1.0);
                        }
                    }

                    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
                                    .updater(new NoOp()).weightInit(WeightInit.DISTRIBUTION)
                                    .dist(new NormalDistribution(0, 1)).convolutionMode(ConvolutionMode.Same).list()
                                    .layer(0, new Convolution1DLayer.Builder().activation(afn).kernelSize(kernel)
                                                    .stride(stride).padding(padding).nIn(convNIn).nOut(convNOut1)
                                                    .build())
                                    .layer(1, new Convolution1DLayer.Builder().activation(afn).kernelSize(kernel)
                                                    .stride(stride).padding(padding).nIn(convNOut1).nOut(convNOut2)
                                                    .build())
                                    .layer(2, new Subsampling1DLayer.Builder(poolingType).kernelSize(kernel)
                                                    .stride(stride).padding(padding).pnorm(pnorm).build())
                                    .layer(3, new RnnOutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
                                                    .activation(Activation.SOFTMAX).nOut(finalNOut).build())
                                    .setInputType(InputType.recurrent(convNIn)).build();

                    String json = conf.toJson();
                    MultiLayerConfiguration c2 = MultiLayerConfiguration.fromJson(json);
                    assertEquals(conf, c2);

                    MultiLayerNetwork net = new MultiLayerNetwork(conf);
                    net.init();

                    String msg = "PoolingType=" + poolingType + ", minibatch=" + minibatchSize + ", activationFn="
                                    + afn + ", kernel = " + kernel;

                    if (PRINT_RESULTS) {
                        System.out.println(msg);
                        for (int j = 0; j < net.getnLayers(); j++)
                            System.out.println("Layer " + j + " # params: " + net.getLayer(j).numParams());
                    }

                    boolean gradOK = GradientCheckUtil.checkGradients(net, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR,
                                    DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels);

                    assertTrue(msg, gradOK);
                }
            }
        }
    }
}