Java PipelineModel.transform方法代码示例

import org.apache.spark.ml.PipelineModel; //导入方法依赖的package包/类
public static void main(String[] args) {
    SparkConf conf = new SparkConf()
            .setAppName("Mnist Classification Pipeline (Java)");
    SparkContext jsc = new SparkContext(conf);
    SQLContext jsql = new SQLContext(jsc);

    String imagesPath = args.length == 2 ? args[0]
            : "file://" + System.getProperty("user.dir") + "/data/train-images-idx3-ubyte";
    String labelsPath = args.length == 2 ? args[1]
            : "file://" + System.getProperty("user.dir") + "/data/train-labels-idx1-ubyte";
    Map<String, String> params = new HashMap<String, String>();
    params.put("imagesPath", imagesPath);
    params.put("labelsPath", labelsPath);
    DataFrame data = jsql.read().format(DefaultSource.class.getName())
            .options(params).load();

    System.out.println("\nLoaded Mnist dataframe:");
    data.show(100);

    DataFrame trainingData = data.sample(false, 0.8, 123);
    DataFrame testData = data.except(trainingData);

    StandardScaler scaler = new StandardScaler()
            .setWithMean(true).setWithStd(true)
            .setInputCol("features")
            .setOutputCol("scaledFeatures");
    NeuralNetworkClassification classification = new NeuralNetworkClassification()
            .setFeaturesCol("scaledFeatures")
            .setConf(getConfiguration());
    Pipeline pipeline = new Pipeline().setStages(new PipelineStage[]{
            scaler, classification});

    System.out.println("\nTraining...");
    PipelineModel model = pipeline.fit(trainingData);

    System.out.println("\nTesting...");
    DataFrame predictions = model.transform(testData);

    System.out.println("\nTest Results:");
    predictions.show(100);
}
 

import org.apache.spark.ml.PipelineModel; //导入方法依赖的package包/类
public static void main(String[] args) {
    SparkConf conf = new SparkConf()
            .setAppName("Iris Classification Pipeline (Java)");
    SparkContext jsc = new SparkContext(conf);
    SQLContext jsql = new SQLContext(jsc);

    String path = args.length == 1 ? args[0]
            : "file://" + System.getProperty("user.dir") + "/data/svmLight/iris_svmLight_0.txt";
    DataFrame data = jsql.read()
            .format(DefaultSource.class.getName())
            .load(path);

    System.out.println("\nLoaded IRIS dataframe:");
    data.show(100);

    // prepare train/test set
    DataFrame trainingData = data.sample(false, 0.6, 11L);
    DataFrame testData = data.except(trainingData);

    // Configure an ML pipeline to train a model. In this example,
    // the pipeline combines Spark ML and DL4J elements.
    StandardScaler scaler = new StandardScaler()
            .setWithMean(true).setWithStd(true)
            .setInputCol("features")
            .setOutputCol("scaledFeatures");
    NeuralNetworkClassification classification = new NeuralNetworkClassification()
            .setFeaturesCol("scaledFeatures")
            .setConf(getConfiguration());
    Pipeline pipeline = new Pipeline().setStages(new PipelineStage[] {
            scaler, classification });

    // Fit the pipeline on training data.
    System.out.println("\nTraining...");
    PipelineModel model = pipeline.fit(trainingData);

    // Make predictions on test data.
    System.out.println("\nTesting...");
    DataFrame predictions = model.transform(testData);

    System.out.println("\nTest Results:");
    predictions.show(100);
}
 

import org.apache.spark.ml.PipelineModel; //导入方法依赖的package包/类
public static void main(String[] args) {
    SparkConf conf = new SparkConf()
            .setAppName("LFW Classification (Java)");
    SparkContext jsc = new SparkContext(conf);
    SQLContext jsql = new SQLContext(jsc);

    if(args.length != 1) {
        System.out.println("usage: run-example ml.JavaLfwClassification <URI>");
        System.out.println("where:\n\tURI: filesystem path to the lFW dataset");
        return;
    }
    String path = args[0];
    DataFrame data = jsql.read()
            .format(DefaultSource.class.getName())
            .load(path);

    // cache all columns upfront
    //data.cache();

    System.out.println("\nLoaded LFW dataframe:");
    data.show(100);

    // prepare train/test set
    DataFrame trainingData = data.sample(false, 0.6, 11L);
    DataFrame testData = data.except(trainingData);

    // Configure an ML pipeline to train a model. In this example,
    // the pipeline combines Spark ML and DL4J elements.
    StringIndexer indexer = new StringIndexer()
            .setInputCol("label")
            .setOutputCol("labelIndex");
    StandardScaler scaler = new StandardScaler()
            .setWithMean(true).setWithStd(true)
            .setInputCol("features").setOutputCol("scaledFeatures");
    NeuralNetworkClassification classification = new NeuralNetworkClassification()
            .setLabelCol("labelIndex")
            .setFeaturesCol("scaledFeatures")
            .setConf(getConfiguration(data));
    Pipeline pipeline = new Pipeline().setStages(new PipelineStage[] {
            indexer, scaler, classification });

    // Fit the pipeline on training data.
    System.out.println("\nTraining...");
    PipelineModel model = pipeline.fit(trainingData);

    // Make predictions on test data.
    System.out.println("\nTesting...");
    DataFrame predictions = model.transform(testData);

    System.out.println("\nTest Results:");
    predictions.show(100);
}
 

import org.apache.spark.ml.PipelineModel; //导入方法依赖的package包/类
public static void main(String[] args) {
    SparkConf conf = new SparkConf().setMaster("spark://babar1.musigma.com:7077")
            .setAppName("Mnist Classification Pipeline (Java)");
    SparkContext jsc = new SparkContext(conf);
    SQLContext jsql = new SQLContext(jsc);

    String imagesPath =  "file://" + System.getProperty("user.home") + "/MNIST/images-idx1-ubyte";
    String labelsPath =  "file://" + System.getProperty("user.home") + "/MNIST/labels-idx1-ubyte";
    Map<String, String> params = new HashMap<>();
    params.put("imagesPath", imagesPath);
    params.put("labelsPath", labelsPath);
    params.put("recordsPerPartition", "400");
    params.put("maxRecords", "2000");

    DataFrame data = jsql.read().format(DefaultSource.class.getName())
            .options(params).load();

    System.out.println("\nLoaded Mnist dataframe:");
    data.show(100);

    DataFrame trainingData = data.sample(false, 0.8, 123);
    DataFrame testData = data.except(trainingData);

    StandardScaler scaler = new StandardScaler()
            .setInputCol("features")
            .setOutputCol("scaledFeatures");
    NeuralNetworkClassification classification = new NeuralNetworkClassification()
            .setFeaturesCol("scaledFeatures")
            .setEpochs(2)
            .setConf(getConfiguration());
    Pipeline pipeline = new Pipeline().setStages(new PipelineStage[]{
            scaler, classification});

    System.out.println("\nTraining...");
    PipelineModel model = pipeline.fit(trainingData);

    System.out.println("\nTesting...");
    DataFrame predictions = model.transform(testData);
    predictions.cache();

    System.out.println("\nTest Results:");
    predictions.show(100);

    Evaluation eval = new Evaluation(outputNum);
    Row[] rows = predictions.select("label","prediction").collect();
    for(int i = 0; i < rows.length; i++) {
        INDArray label = FeatureUtil.toOutcomeVector((int) rows[i].getDouble(0), outputNum);
        INDArray prediction = FeatureUtil.toOutcomeVector((int) rows[i].getDouble(1), outputNum);
        eval.eval(label, prediction);
    }

    System.out.println(eval.stats());
}
 

import org.apache.spark.ml.PipelineModel; //导入方法依赖的package包/类
public static void main(String[] args) {
    SparkConf conf = new SparkConf().setMaster("local[*]")
            .setAppName("Cards Identification Pipeline (Java)");
    SparkContext jsc = new SparkContext(conf);
    SQLContext jsql = new SQLContext(jsc);

    String imagesPath =  "file://" + System.getProperty("user.home") + "/MNIST/images-idx1-ubyte";
    String labelsPath =  "file://" + System.getProperty("user.home") + "/MNIST/labels-idx1-ubyte";
    Map<String, String> params = new HashMap<>();
    params.put("imagesPath", imagesPath);
    params.put("labelsPath", labelsPath);
    params.put("recordsPerPartition", "400");
    params.put("maxRecords", "2000");

    DataFrame data = jsql.read().format(DefaultSource.class.getName())
            .options(params).load();

    System.out.println("\nLoaded Card Images dataframe:");
    data.show(100);

    DataFrame trainingData = data.sample(false, 0.8, 123);
    DataFrame testData = data.except(trainingData);

    StandardScaler scaler = new StandardScaler()
            .setInputCol("features")
            .setOutputCol("scaledFeatures");
    NeuralNetworkClassification classification = new NeuralNetworkClassification()
            .setFeaturesCol("scaledFeatures")
            .setEpochs(2)
            .setConf(getConfiguration());
    Pipeline pipeline = new Pipeline().setStages(new PipelineStage[]{
            scaler, classification});

    System.out.println("\nTraining...");
    PipelineModel model = pipeline.fit(trainingData);

    System.out.println("\nTesting...");
    DataFrame predictions = model.transform(testData);
    predictions.cache();

    System.out.println("\nTest Results:");
    predictions.show(100);

    Evaluation eval = new Evaluation(outputNum);
    Row[] rows = predictions.select("label","prediction").collect();
    for(int i = 0; i < rows.length; i++) {
        INDArray label = FeatureUtil.toOutcomeVector((int) rows[i].getDouble(0), outputNum);
        INDArray prediction = FeatureUtil.toOutcomeVector((int) rows[i].getDouble(1), outputNum);
        eval.eval(label, prediction);
    }

    System.out.println(eval.stats());
}