Java Activation.SOFTMAX属性代码示例

@Test
public void gradientCheckMaskingOutputSimple() {

    int timeSeriesLength = 5;
    boolean[][] mask = new boolean[5][0];
    mask[0] = new boolean[] {true, true, true, true, true}; //No masking
    mask[1] = new boolean[] {false, true, true, true, true}; //mask first output time step
    mask[2] = new boolean[] {false, false, false, false, true}; //time series classification: mask all but last
    mask[3] = new boolean[] {false, false, true, false, true}; //time series classification w/ variable length TS
    mask[4] = new boolean[] {true, true, true, false, true}; //variable length TS

    int nIn = 4;
    int layerSize = 3;

    GradientCheckSimpleScenario[] scenarios = new GradientCheckSimpleScenario[] {
                    new GradientCheckSimpleScenario(LossFunctions.LossFunction.MCXENT.getILossFunction(),
                                    Activation.SOFTMAX, 2, 2),
                    new GradientCheckSimpleScenario(LossMixtureDensity.builder().gaussians(2).labelWidth(3).build(),
                                    Activation.TANH, 10, 3),
                    new GradientCheckSimpleScenario(LossMixtureDensity.builder().gaussians(2).labelWidth(4).build(),
                                    Activation.IDENTITY, 12, 4),
                    new GradientCheckSimpleScenario(LossFunctions.LossFunction.L2.getILossFunction(),
                                    Activation.SOFTMAX, 2, 2)};

    for (GradientCheckSimpleScenario s : scenarios) {

        Random r = new Random(12345L);
        INDArray input = Nd4j.zeros(1, nIn, timeSeriesLength);
        for (int m = 0; m < 1; m++) {
            for (int j = 0; j < nIn; j++) {
                for (int k = 0; k < timeSeriesLength; k++) {
                    input.putScalar(new int[] {m, j, k}, r.nextDouble() - 0.5);
                }
            }
        }

        INDArray labels = Nd4j.zeros(1, s.labelWidth, timeSeriesLength);
        for (int m = 0; m < 1; m++) {
            for (int j = 0; j < timeSeriesLength; j++) {
                int idx = r.nextInt(s.labelWidth);
                labels.putScalar(new int[] {m, idx, j}, 1.0f);
            }
        }

        for (int i = 0; i < mask.length; i++) {

            //Create mask array:
            INDArray maskArr = Nd4j.create(1, timeSeriesLength);
            for (int j = 0; j < mask[i].length; j++) {
                maskArr.putScalar(new int[] {0, j}, mask[i][j] ? 1.0 : 0.0);
            }

            MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345L)
                            .list()
                            .layer(0, new GravesLSTM.Builder().nIn(nIn).nOut(layerSize)
                                            .weightInit(WeightInit.DISTRIBUTION).dist(new NormalDistribution(0, 1))
                                            .updater(new NoOp()).build())
                            .layer(1, new RnnOutputLayer.Builder(s.lf).activation(s.act).nIn(layerSize).nOut(s.nOut)
                                            .weightInit(WeightInit.DISTRIBUTION).dist(new NormalDistribution(0, 1))
                                            .updater(new NoOp()).build())
                            .pretrain(false).backprop(true).build();
            MultiLayerNetwork mln = new MultiLayerNetwork(conf);
            mln.init();

            boolean gradOK = GradientCheckUtil.checkGradients(mln, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR,
                            DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels, null, maskArr);

            String msg = "gradientCheckMaskingOutputSimple() - timeSeriesLength=" + timeSeriesLength
                            + ", miniBatchSize=" + 1;
            assertTrue(msg, gradOK);
        }
    }
}
 

@Test
public void testPerOutputMaskingMLP() {
    int nIn = 6;
    int layerSize = 4;

    INDArray mask1 = Nd4j.create(new double[] {1, 0, 0, 1, 0});
    INDArray mask3 = Nd4j.create(new double[][] {{1, 1, 1, 1, 1}, {0, 1, 0, 1, 0}, {1, 0, 0, 1, 1}});
    INDArray[] labelMasks = new INDArray[] {mask1, mask3};

    ILossFunction[] lossFunctions = new ILossFunction[] {new LossBinaryXENT(),
                    //                new LossCosineProximity(),    //Doesn't support per-output masking, as it doesn't make sense for cosine proximity
                    new LossHinge(), new LossKLD(), new LossKLD(), new LossL1(), new LossL2(), new LossMAE(),
                    new LossMAE(), new LossMAPE(), new LossMAPE(),
                    //                new LossMCXENT(),             //Per output masking on MCXENT+Softmax: not yet supported
                    new LossMCXENT(), new LossMSE(), new LossMSE(), new LossMSLE(), new LossMSLE(),
                    new LossNegativeLogLikelihood(), new LossPoisson(), new LossSquaredHinge()};

    Activation[] act = new Activation[] {Activation.SIGMOID, //XENT
                    //                Activation.TANH,
                    Activation.TANH, //Hinge
                    Activation.SIGMOID, //KLD
                    Activation.SOFTMAX, //KLD + softmax
                    Activation.TANH, //L1
                    Activation.TANH, //L2
                    Activation.TANH, //MAE
                    Activation.SOFTMAX, //MAE + softmax
                    Activation.TANH, //MAPE
                    Activation.SOFTMAX, //MAPE + softmax
                    //                Activation.SOFTMAX, //MCXENT + softmax: see comment above
                    Activation.SIGMOID, //MCXENT + sigmoid
                    Activation.TANH, //MSE
                    Activation.SOFTMAX, //MSE + softmax
                    Activation.SIGMOID, //MSLE - needs positive labels/activations (due to log)
                    Activation.SOFTMAX, //MSLE + softmax
                    Activation.SIGMOID, //NLL
                    Activation.SIGMOID, //Poisson
                    Activation.TANH //Squared hinge
    };

    for (INDArray labelMask : labelMasks) {

        int minibatch = labelMask.size(0);
        int nOut = labelMask.size(1);

        for (int i = 0; i < lossFunctions.length; i++) {
            ILossFunction lf = lossFunctions[i];
            Activation a = act[i];


            MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().updater(new NoOp())
                            .weightInit(WeightInit.DISTRIBUTION).dist(new NormalDistribution(0, 1)).seed(12345)
                            .list()
                            .layer(0, new DenseLayer.Builder().nIn(nIn).nOut(layerSize).activation(Activation.TANH)
                                            .build())
                            .layer(1, new OutputLayer.Builder().nIn(layerSize).nOut(nOut).lossFunction(lf)
                                            .activation(a).build())
                            .build();

            MultiLayerNetwork net = new MultiLayerNetwork(conf);
            net.init();

            INDArray[] fl = LossFunctionGradientCheck.getFeaturesAndLabels(lf, minibatch, nIn, nOut, 12345);
            INDArray features = fl[0];
            INDArray labels = fl[1];

            String msg = "testPerOutputMaskingMLP(): maskShape = " + Arrays.toString(labelMask.shape())
                            + ", loss function = " + lf + ", activation = " + a;

            System.out.println(msg);

            boolean gradOK = GradientCheckUtil.checkGradients(net, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR,
                            DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, features, labels, null, labelMask);

            assertTrue(msg, gradOK);
        }
    }
}
 

@Test
public void testGradientCNNMLN() {
    //Parameterized test, testing combinations of:
    // (a) activation function
    // (b) Whether to test at random initialization, or after some learning (i.e., 'characteristic mode of operation')
    // (c) Loss function (with specified output activations)
    Activation[] activFns = {Activation.SIGMOID, Activation.TANH};
    boolean[] characteristic = {false, true}; //If true: run some backprop steps first

    LossFunctions.LossFunction[] lossFunctions =
            {LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD, LossFunctions.LossFunction.MSE};
    Activation[] outputActivations = {Activation.SOFTMAX, Activation.TANH}; //i.e., lossFunctions[i] used with outputActivations[i] here

    DataSet ds = new IrisDataSetIterator(150, 150).next();
    ds.normalizeZeroMeanZeroUnitVariance();
    INDArray input = ds.getFeatureMatrix();
    INDArray labels = ds.getLabels();

    for (Activation afn : activFns) {
        for (boolean doLearningFirst : characteristic) {
            for (int i = 0; i < lossFunctions.length; i++) {
                LossFunctions.LossFunction lf = lossFunctions[i];
                Activation outputActivation = outputActivations[i];

                MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder()
                        .optimizationAlgo(OptimizationAlgorithm.CONJUGATE_GRADIENT).updater(new NoOp())
                        .weightInit(WeightInit.XAVIER).seed(12345L).list()
                        .layer(0, new ConvolutionLayer.Builder(1, 1).nOut(6).activation(afn).build())
                        .layer(1, new OutputLayer.Builder(lf).activation(outputActivation).nOut(3).build())
                        .setInputType(InputType.convolutionalFlat(1, 4, 1)).pretrain(false).backprop(true);

                MultiLayerConfiguration conf = builder.build();

                MultiLayerNetwork mln = new MultiLayerNetwork(conf);
                mln.init();
                String name = new Object() {
                }.getClass().getEnclosingMethod().getName();

                if (doLearningFirst) {
                    //Run a number of iterations of learning
                    mln.setInput(ds.getFeatures());
                    mln.setLabels(ds.getLabels());
                    mln.computeGradientAndScore();
                    double scoreBefore = mln.score();
                    for (int j = 0; j < 10; j++)
                        mln.fit(ds);
                    mln.computeGradientAndScore();
                    double scoreAfter = mln.score();
                    //Can't test in 'characteristic mode of operation' if not learning
                    String msg = name + " - score did not (sufficiently) decrease during learning - activationFn="
                            + afn + ", lossFn=" + lf + ", outputActivation=" + outputActivation
                            + ", doLearningFirst= " + doLearningFirst + " (before=" + scoreBefore
                            + ", scoreAfter=" + scoreAfter + ")";
                    assertTrue(msg, scoreAfter < 0.8 * scoreBefore);
                }

                if (PRINT_RESULTS) {
                    System.out.println(name + " - activationFn=" + afn + ", lossFn=" + lf + ", outputActivation="
                            + outputActivation + ", doLearningFirst=" + doLearningFirst);
                    for (int j = 0; j < mln.getnLayers(); j++)
                        System.out.println("Layer " + j + " # params: " + mln.getLayer(j).numParams());
                }

                boolean gradOK = GradientCheckUtil.checkGradients(mln, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR,
                        DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels);

                assertTrue(gradOK);
            }
        }
    }
}
 

@Test
public void testVaeAsMLP() {
    //Post pre-training: a VAE can be used as a MLP, by taking the mean value from p(z|x) as the output
    //This gradient check tests this part

    Activation[] activFns = {Activation.IDENTITY, Activation.TANH, Activation.IDENTITY, Activation.TANH, Activation.IDENTITY, Activation.TANH};

    LossFunction[] lossFunctions = {LossFunction.MCXENT, LossFunction.MCXENT, LossFunction.MSE, LossFunction.MSE, LossFunction.MCXENT, LossFunction.MSE};
    Activation[] outputActivations = {Activation.SOFTMAX, Activation.SOFTMAX, Activation.TANH, Activation.TANH, Activation.SOFTMAX, Activation.TANH};

    //use l2vals[i] with l1vals[i]
    double[] l2vals = {0.4, 0.0, 0.4, 0.4, 0.0, 0.0};
    double[] l1vals = {0.0, 0.0, 0.5, 0.0, 0.0, 0.5};
    double[] biasL2 = {0.0, 0.0, 0.0, 0.2, 0.0, 0.4};
    double[] biasL1 = {0.0, 0.0, 0.6, 0.0, 0.0, 0.0};

    int[][] encoderLayerSizes = new int[][] {{5}, {5}, {5, 6}, {5, 6}, {5}, {5, 6}};
    int[][] decoderLayerSizes = new int[][] {{6}, {7, 8}, {6}, {7, 8}, {6}, {7, 8}};

    int[] minibatches = new int[]{1,5,4,3,1,4};

    Nd4j.getRandom().setSeed(12345);
    for( int i=0; i<activFns.length; i++ ){
        LossFunction lf = lossFunctions[i];
        Activation outputActivation = outputActivations[i];
        double l2 = l2vals[i];
        double l1 = l1vals[i];
        int[] encoderSizes = encoderLayerSizes[i];
        int[] decoderSizes = decoderLayerSizes[i];
        int minibatch = minibatches[i];
        INDArray input = Nd4j.rand(minibatch, 4);
        INDArray labels = Nd4j.create(minibatch, 3);
        for (int j = 0; j < minibatch; j++) {
            labels.putScalar(j, j % 3, 1.0);
        }
        Activation afn = activFns[i];

        MultiLayerConfiguration conf =
                new NeuralNetConfiguration.Builder().l2(l2).l1(l1)
                        .updater(new NoOp())
                        .l2Bias(biasL2[i]).l1Bias(biasL1[i])
                        .updater(new NoOp()).seed(12345L).list()
                        .layer(0, new VariationalAutoencoder.Builder().nIn(4)
                                .nOut(3).encoderLayerSizes(encoderSizes)
                                .decoderLayerSizes(decoderSizes)
                                .weightInit(WeightInit.DISTRIBUTION)
                                .dist(new NormalDistribution(0, 1))
                                .activation(afn)
                                .build())
                        .layer(1, new OutputLayer.Builder(lf)
                                .activation(outputActivation).nIn(3).nOut(3)
                                .weightInit(WeightInit.DISTRIBUTION)
                                .dist(new NormalDistribution(0, 1))
                                .build())
                        .pretrain(false).backprop(true).build();

        MultiLayerNetwork mln = new MultiLayerNetwork(conf);
        mln.init();

        String msg = "testVaeAsMLP() - activationFn=" + afn + ", lossFn=" + lf
                + ", outputActivation=" + outputActivation + ", encLayerSizes = "
                + Arrays.toString(encoderSizes) + ", decLayerSizes = "
                + Arrays.toString(decoderSizes) + ", l2=" + l2 + ", l1=" + l1;
        if (PRINT_RESULTS) {
            System.out.println(msg);
            for (int j = 0; j < mln.getnLayers(); j++)
                System.out.println("Layer " + j + " # params: " + mln.getLayer(j).numParams());
        }

        boolean gradOK = GradientCheckUtil.checkGradients(mln, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR,
                DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input,
                labels);
        assertTrue(msg, gradOK);
    }
}