Java FeatureUtil.toOutcomeVector方法代码示例

import org.nd4j.linalg.util.FeatureUtil; //导入方法依赖的package包/类
public Pair<INDArray, opencv_core.Mat> convertMat(byte[] byteFeature) {
    INDArray label = FeatureUtil.toOutcomeVector(byteFeature[0], NUM_LABELS);; // first value in the 3073 byte array
    opencv_core.Mat image = new opencv_core.Mat(HEIGHT, WIDTH, CV_8UC(CHANNELS)); // feature are 3072
    ByteBuffer imageData = image.createBuffer();

    for (int i = 0; i < HEIGHT * WIDTH; i++) {
        imageData.put(3 * i, byteFeature[i + 1 + 2 * HEIGHT * WIDTH]); // blue
        imageData.put(3 * i + 1, byteFeature[i + 1 + HEIGHT * WIDTH]); // green
        imageData.put(3 * i + 2, byteFeature[i + 1]); // red
    }
    //        if (useSpecialPreProcessCifar) {
    //            image = convertCifar(image);
    //        }

    return new Pair<>(label, image);
}
 

import org.nd4j.linalg.util.FeatureUtil; //导入方法依赖的package包/类
@Test
public void testStringListLabels() {
    INDArray trueOutcome = FeatureUtil.toOutcomeVector(0, 2);
    INDArray predictedOutcome = FeatureUtil.toOutcomeVector(0, 2);

    List<String> labelsList = new ArrayList<>();
    labelsList.add("hobbs");
    labelsList.add("cal");

    Evaluation eval = new Evaluation(labelsList);

    eval.eval(trueOutcome, predictedOutcome);
    assertEquals(1, eval.classCount(0));
    assertEquals(labelsList.get(0), eval.getClassLabel(0));

}
 

import org.nd4j.linalg.util.FeatureUtil; //导入方法依赖的package包/类
@Test
public void testStringHashLabels() {
    INDArray trueOutcome = FeatureUtil.toOutcomeVector(0, 2);
    INDArray predictedOutcome = FeatureUtil.toOutcomeVector(0, 2);

    Map<Integer, String> labelsMap = new HashMap<>();
    labelsMap.put(0, "hobbs");
    labelsMap.put(1, "cal");

    Evaluation eval = new Evaluation(labelsMap);

    eval.eval(trueOutcome, predictedOutcome);
    assertEquals(1, eval.classCount(0));
    assertEquals(labelsMap.get(0), eval.getClassLabel(0));

}
 

import org.nd4j.linalg.util.FeatureUtil; //导入方法依赖的package包/类
@Override
public DataSet convert(Collection<Collection<Writable>> records, int numLabels) {
    //all but last label
    DataSet ret = new DataSet(Nd4j.create(records.size(), records.iterator().next().size() - 1),
                    Nd4j.create(records.size(), numLabels));
    //  INDArray ret = Nd4j.create(records.size(),records.iterator().next().size() - 1);
    int count = 0;
    for (Collection<Writable> record : records) {
        List<Writable> list;
        if (record instanceof List) {
            list = (List<Writable>) record;
        } else
            list = new ArrayList<>(record);
        DataSet d = new DataSet(Nd4j.create(record.size() - 1),
                        FeatureUtil.toOutcomeVector(list.get(list.size() - 1).toInt(), numLabels));
        ret.addRow(d, count++);

    }


    return ret;
}
 

import org.nd4j.linalg.util.FeatureUtil; //导入方法依赖的package包/类
/**
 * Sets the outcome of a particular example
 * @param example the example to applyTransformToDestination
 * @param label the label of the outcome
 */
@Override
public void setOutcome(int example, int label) {
    if(example > numExamples())
        throw new IllegalArgumentException("No example at " + example);
    if(label > numOutcomes() || label < 0)
        throw new IllegalArgumentException("Illegal label");

    INDArray outcome = FeatureUtil.toOutcomeVector(label, numOutcomes());
    getLabels().putRow(example,outcome);
}
 

import org.nd4j.linalg.util.FeatureUtil; //导入方法依赖的package包/类
/**
 * Sets the outcome of a particular example
 *
 * @param example the example to transform
 * @param label   the label of the outcome
 */
@Override
public void setOutcome(int example, int label) {
    if (example > numExamples())
        throw new IllegalArgumentException("No example at " + example);
    if (label > numOutcomes() || label < 0)
        throw new IllegalArgumentException("Illegal label");

    INDArray outcome = FeatureUtil.toOutcomeVector(label, numOutcomes());
    getLabels().putRow(example, outcome);
}
 

import org.nd4j.linalg.util.FeatureUtil; //导入方法依赖的package包/类
public DataSet getDataFor(int i) {
    File image = new File(images.get(i));
    int outcome = outcomes.indexOf(image.getParentFile().getAbsolutePath());
    try {
        return new DataSet(loader.asRowVector(image), FeatureUtil.toOutcomeVector(outcome, outcomes.size()));
    } catch (Exception e) {
        throw new IllegalStateException("Unable to getFromOrigin data for image " + i + " for path " + images.get(i));
    }
}
 

import org.nd4j.linalg.util.FeatureUtil; //导入方法依赖的package包/类
/**
 * Vectorizes the passed in text treating it as one document
 *
 * @param text  the text to vectorize
 * @param label the label of the text
 * @return a dataset with a transform of weights(relative to impl; could be word counts or tfidf scores)
 */
@Override
public DataSet vectorize(String text, String label) {
    INDArray input = transform(text);
    INDArray labelMatrix = FeatureUtil.toOutcomeVector(labelsSource.indexOf(label), labelsSource.size());

    return new DataSet(input, labelMatrix);
}
 

import org.nd4j.linalg.util.FeatureUtil; //导入方法依赖的package包/类
@Override
public DataSet vectorize(String text, String label) {
    INDArray input = transform(text);
    INDArray labelMatrix = FeatureUtil.toOutcomeVector(labelsSource.indexOf(label), labelsSource.size());

    return new DataSet(input, labelMatrix);
}
 

import org.nd4j.linalg.util.FeatureUtil; //导入方法依赖的package包/类
@Test
public void testEval() {
    int classNum = 5;
    Evaluation eval = new Evaluation(classNum);

    // Testing the edge case when some classes do not have true positive
    INDArray trueOutcome = FeatureUtil.toOutcomeVector(0, 5); //[1,0,0,0,0]
    INDArray predictedOutcome = FeatureUtil.toOutcomeVector(0, 5); //[1,0,0,0,0]
    eval.eval(trueOutcome, predictedOutcome);
    assertEquals(1, eval.classCount(0));
    assertEquals(1.0, eval.f1(), 1e-1);

    // Testing more than one sample. eval() does not reset the Evaluation instance
    INDArray trueOutcome2 = FeatureUtil.toOutcomeVector(1, 5); //[0,1,0,0,0]
    INDArray predictedOutcome2 = FeatureUtil.toOutcomeVector(0, 5); //[1,0,0,0,0]
    eval.eval(trueOutcome2, predictedOutcome2);
    // Verified with sklearn in Python
    // from sklearn.metrics import classification_report
    // classification_report(['a', 'a'], ['a', 'b'], labels=['a', 'b', 'c', 'd', 'e'])
    assertEquals(eval.f1(), 0.6, 1e-1);
    // The first entry is 0 label
    assertEquals(1, eval.classCount(0));
    // The first entry is 1 label
    assertEquals(1, eval.classCount(1));
    // Class 0: one positive, one negative -> (one true positive, one false positive); no true/false negatives
    assertEquals(1, eval.positive().get(0), 0);
    assertEquals(1, eval.negative().get(0), 0);
    assertEquals(1, eval.truePositives().get(0), 0);
    assertEquals(1, eval.falsePositives().get(0), 0);
    assertEquals(0, eval.trueNegatives().get(0), 0);
    assertEquals(0, eval.falseNegatives().get(0), 0);


    // The rest are negative
    assertEquals(1, eval.negative().get(0), 0);
    // 2 rows and only the first is correct
    assertEquals(0.5, eval.accuracy(), 0);
}
 

import org.nd4j.linalg.util.FeatureUtil; //导入方法依赖的package包/类
/**
 *
 * @param point
 * @param numPossibleLabels
 * @return {@link DataSet}
 */
private static DataSet fromLabeledPoint(LabeledPoint point, int numPossibleLabels) {
    Vector features = point.features();
    double label = point.label();
    return new DataSet(Nd4j.create(features.toArray()),
                    FeatureUtil.toOutcomeVector((int) label, numPossibleLabels));
}
 

import org.nd4j.linalg.util.FeatureUtil; //导入方法依赖的package包/类
public static void main(String[] args) {
    SparkConf conf = new SparkConf().setMaster("spark://babar1.musigma.com:7077")
            .setAppName("Mnist Classification Pipeline (Java)");
    SparkContext jsc = new SparkContext(conf);
    SQLContext jsql = new SQLContext(jsc);

    String imagesPath =  "file://" + System.getProperty("user.home") + "/MNIST/images-idx1-ubyte";
    String labelsPath =  "file://" + System.getProperty("user.home") + "/MNIST/labels-idx1-ubyte";
    Map<String, String> params = new HashMap<>();
    params.put("imagesPath", imagesPath);
    params.put("labelsPath", labelsPath);
    params.put("recordsPerPartition", "400");
    params.put("maxRecords", "2000");

    DataFrame data = jsql.read().format(DefaultSource.class.getName())
            .options(params).load();

    System.out.println("\nLoaded Mnist dataframe:");
    data.show(100);

    DataFrame trainingData = data.sample(false, 0.8, 123);
    DataFrame testData = data.except(trainingData);

    StandardScaler scaler = new StandardScaler()
            .setInputCol("features")
            .setOutputCol("scaledFeatures");
    NeuralNetworkClassification classification = new NeuralNetworkClassification()
            .setFeaturesCol("scaledFeatures")
            .setEpochs(2)
            .setConf(getConfiguration());
    Pipeline pipeline = new Pipeline().setStages(new PipelineStage[]{
            scaler, classification});

    System.out.println("\nTraining...");
    PipelineModel model = pipeline.fit(trainingData);

    System.out.println("\nTesting...");
    DataFrame predictions = model.transform(testData);
    predictions.cache();

    System.out.println("\nTest Results:");
    predictions.show(100);

    Evaluation eval = new Evaluation(outputNum);
    Row[] rows = predictions.select("label","prediction").collect();
    for(int i = 0; i < rows.length; i++) {
        INDArray label = FeatureUtil.toOutcomeVector((int) rows[i].getDouble(0), outputNum);
        INDArray prediction = FeatureUtil.toOutcomeVector((int) rows[i].getDouble(1), outputNum);
        eval.eval(label, prediction);
    }

    System.out.println(eval.stats());
}
 

import org.nd4j.linalg.util.FeatureUtil; //导入方法依赖的package包/类
public static void main(String[] args) {
    SparkConf conf = new SparkConf().setMaster("local[*]")
            .setAppName("Cards Identification Pipeline (Java)");
    SparkContext jsc = new SparkContext(conf);
    SQLContext jsql = new SQLContext(jsc);

    String imagesPath =  "file://" + System.getProperty("user.home") + "/MNIST/images-idx1-ubyte";
    String labelsPath =  "file://" + System.getProperty("user.home") + "/MNIST/labels-idx1-ubyte";
    Map<String, String> params = new HashMap<>();
    params.put("imagesPath", imagesPath);
    params.put("labelsPath", labelsPath);
    params.put("recordsPerPartition", "400");
    params.put("maxRecords", "2000");

    DataFrame data = jsql.read().format(DefaultSource.class.getName())
            .options(params).load();

    System.out.println("\nLoaded Card Images dataframe:");
    data.show(100);

    DataFrame trainingData = data.sample(false, 0.8, 123);
    DataFrame testData = data.except(trainingData);

    StandardScaler scaler = new StandardScaler()
            .setInputCol("features")
            .setOutputCol("scaledFeatures");
    NeuralNetworkClassification classification = new NeuralNetworkClassification()
            .setFeaturesCol("scaledFeatures")
            .setEpochs(2)
            .setConf(getConfiguration());
    Pipeline pipeline = new Pipeline().setStages(new PipelineStage[]{
            scaler, classification});

    System.out.println("\nTraining...");
    PipelineModel model = pipeline.fit(trainingData);

    System.out.println("\nTesting...");
    DataFrame predictions = model.transform(testData);
    predictions.cache();

    System.out.println("\nTest Results:");
    predictions.show(100);

    Evaluation eval = new Evaluation(outputNum);
    Row[] rows = predictions.select("label","prediction").collect();
    for(int i = 0; i < rows.length; i++) {
        INDArray label = FeatureUtil.toOutcomeVector((int) rows[i].getDouble(0), outputNum);
        INDArray prediction = FeatureUtil.toOutcomeVector((int) rows[i].getDouble(1), outputNum);
        eval.eval(label, prediction);
    }

    System.out.println(eval.stats());
}
 

import org.nd4j.linalg.util.FeatureUtil; //导入方法依赖的package包/类
public DataSet fromImageFile(int label,File image) throws Exception {
    INDArray outcome = FeatureUtil.toOutcomeVector(label, numNames);
    INDArray image2 = ArrayUtil.toNDArray(loader.flattenedImageFromFile(image));
    return new DataSet(image2,outcome);
}
 

import org.nd4j.linalg.util.FeatureUtil; //导入方法依赖的package包/类
@Override
public Tuple2<Double, DataSet> call(Tuple2<Text, BytesWritable> inputTuple) throws Exception {
    int lenFeatureVector = 0;

    if (numPossibleLabels >= 1) {
        lenFeatureVector = byteFileLen - 1;
        if (labelIndex < 0)
            labelIndex = byteFileLen - 1;
    }

    InputStream inputStream = new DataInputStream(new ByteArrayInputStream(inputTuple._2().getBytes()));

    int batchNumCount = 0;
    byte[] byteFeature = new byte[byteFileLen];
    List<DataSet> dataSets = new ArrayList<>();
    INDArray label;
    int featureCount;

    try {
        INDArray featureVector = Nd4j.create(lenFeatureVector);
        while ((inputStream.read(byteFeature)) != -1 && batchNumCount != batchSize) {
            featureCount = 0;
            label = FeatureUtil.toOutcomeVector(byteFeature[labelIndex], numPossibleLabels);
            for (int j = 1; j <= featureVector.length(); j++)
                featureVector.putScalar(featureCount++, byteFeature[j]);
            dataSets.add(new DataSet(featureVector, label));
            batchNumCount++;
            byteFeature = new byte[byteFileLen];
            featureVector = Nd4j.create(lenFeatureVector);
        }
    } catch (IOException e) {
        e.printStackTrace();
    }

    List<INDArray> inputs = new ArrayList<>();
    List<INDArray> labels = new ArrayList<>();

    for (DataSet data : dataSets) {
        inputs.add(data.getFeatureMatrix());
        labels.add(data.getLabels());
    }

    DataSet ds = new DataSet(Nd4j.vstack(inputs.toArray(new INDArray[0])),
                    Nd4j.vstack(labels.toArray(new INDArray[0])));
    if (preProcessor != null)
        preProcessor.preProcess(ds);
    return new Tuple2<>((double) batchNumCount, ds);

}