当前位置: 首页>>代码示例>>Java>>正文


Java PipelineModel.transform方法代码示例

本文整理汇总了Java中org.apache.spark.ml.PipelineModel.transform方法的典型用法代码示例。如果您正苦于以下问题:Java PipelineModel.transform方法的具体用法?Java PipelineModel.transform怎么用?Java PipelineModel.transform使用的例子?那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在org.apache.spark.ml.PipelineModel的用法示例。


在下文中一共展示了PipelineModel.transform方法的5个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Java代码示例。

示例1: main

import org.apache.spark.ml.PipelineModel; //导入方法依赖的package包/类
public static void main(String[] args) {
    SparkConf conf = new SparkConf()
            .setAppName("Mnist Classification Pipeline (Java)");
    SparkContext jsc = new SparkContext(conf);
    SQLContext jsql = new SQLContext(jsc);

    String imagesPath = args.length == 2 ? args[0]
            : "file://" + System.getProperty("user.dir") + "/data/train-images-idx3-ubyte";
    String labelsPath = args.length == 2 ? args[1]
            : "file://" + System.getProperty("user.dir") + "/data/train-labels-idx1-ubyte";
    Map<String, String> params = new HashMap<String, String>();
    params.put("imagesPath", imagesPath);
    params.put("labelsPath", labelsPath);
    DataFrame data = jsql.read().format(DefaultSource.class.getName())
            .options(params).load();

    System.out.println("\nLoaded Mnist dataframe:");
    data.show(100);

    DataFrame trainingData = data.sample(false, 0.8, 123);
    DataFrame testData = data.except(trainingData);

    StandardScaler scaler = new StandardScaler()
            .setWithMean(true).setWithStd(true)
            .setInputCol("features")
            .setOutputCol("scaledFeatures");
    NeuralNetworkClassification classification = new NeuralNetworkClassification()
            .setFeaturesCol("scaledFeatures")
            .setConf(getConfiguration());
    Pipeline pipeline = new Pipeline().setStages(new PipelineStage[]{
            scaler, classification});

    System.out.println("\nTraining...");
    PipelineModel model = pipeline.fit(trainingData);

    System.out.println("\nTesting...");
    DataFrame predictions = model.transform(testData);

    System.out.println("\nTest Results:");
    predictions.show(100);
}
 
开发者ID:javadba,项目名称:dl4j-spark-ml-examples,代码行数:42,代码来源:JavaMnistClassification.java

示例2: main

import org.apache.spark.ml.PipelineModel; //导入方法依赖的package包/类
public static void main(String[] args) {
    SparkConf conf = new SparkConf()
            .setAppName("Iris Classification Pipeline (Java)");
    SparkContext jsc = new SparkContext(conf);
    SQLContext jsql = new SQLContext(jsc);

    String path = args.length == 1 ? args[0]
            : "file://" + System.getProperty("user.dir") + "/data/svmLight/iris_svmLight_0.txt";
    DataFrame data = jsql.read()
            .format(DefaultSource.class.getName())
            .load(path);

    System.out.println("\nLoaded IRIS dataframe:");
    data.show(100);

    // prepare train/test set
    DataFrame trainingData = data.sample(false, 0.6, 11L);
    DataFrame testData = data.except(trainingData);

    // Configure an ML pipeline to train a model. In this example,
    // the pipeline combines Spark ML and DL4J elements.
    StandardScaler scaler = new StandardScaler()
            .setWithMean(true).setWithStd(true)
            .setInputCol("features")
            .setOutputCol("scaledFeatures");
    NeuralNetworkClassification classification = new NeuralNetworkClassification()
            .setFeaturesCol("scaledFeatures")
            .setConf(getConfiguration());
    Pipeline pipeline = new Pipeline().setStages(new PipelineStage[] {
            scaler, classification });

    // Fit the pipeline on training data.
    System.out.println("\nTraining...");
    PipelineModel model = pipeline.fit(trainingData);

    // Make predictions on test data.
    System.out.println("\nTesting...");
    DataFrame predictions = model.transform(testData);

    System.out.println("\nTest Results:");
    predictions.show(100);
}
 
开发者ID:javadba,项目名称:dl4j-spark-ml-examples,代码行数:43,代码来源:JavaIrisClassification.java

示例3: main

import org.apache.spark.ml.PipelineModel; //导入方法依赖的package包/类
public static void main(String[] args) {
    SparkConf conf = new SparkConf()
            .setAppName("LFW Classification (Java)");
    SparkContext jsc = new SparkContext(conf);
    SQLContext jsql = new SQLContext(jsc);

    if(args.length != 1) {
        System.out.println("usage: run-example ml.JavaLfwClassification <URI>");
        System.out.println("where:\n\tURI: filesystem path to the lFW dataset");
        return;
    }
    String path = args[0];
    DataFrame data = jsql.read()
            .format(DefaultSource.class.getName())
            .load(path);

    // cache all columns upfront
    //data.cache();

    System.out.println("\nLoaded LFW dataframe:");
    data.show(100);

    // prepare train/test set
    DataFrame trainingData = data.sample(false, 0.6, 11L);
    DataFrame testData = data.except(trainingData);

    // Configure an ML pipeline to train a model. In this example,
    // the pipeline combines Spark ML and DL4J elements.
    StringIndexer indexer = new StringIndexer()
            .setInputCol("label")
            .setOutputCol("labelIndex");
    StandardScaler scaler = new StandardScaler()
            .setWithMean(true).setWithStd(true)
            .setInputCol("features").setOutputCol("scaledFeatures");
    NeuralNetworkClassification classification = new NeuralNetworkClassification()
            .setLabelCol("labelIndex")
            .setFeaturesCol("scaledFeatures")
            .setConf(getConfiguration(data));
    Pipeline pipeline = new Pipeline().setStages(new PipelineStage[] {
            indexer, scaler, classification });

    // Fit the pipeline on training data.
    System.out.println("\nTraining...");
    PipelineModel model = pipeline.fit(trainingData);

    // Make predictions on test data.
    System.out.println("\nTesting...");
    DataFrame predictions = model.transform(testData);

    System.out.println("\nTest Results:");
    predictions.show(100);
}
 
开发者ID:javadba,项目名称:dl4j-spark-ml-examples,代码行数:53,代码来源:JavaLfwClassification.java

示例4: main

import org.apache.spark.ml.PipelineModel; //导入方法依赖的package包/类
public static void main(String[] args) {
    SparkConf conf = new SparkConf().setMaster("spark://babar1.musigma.com:7077")
            .setAppName("Mnist Classification Pipeline (Java)");
    SparkContext jsc = new SparkContext(conf);
    SQLContext jsql = new SQLContext(jsc);

    String imagesPath =  "file://" + System.getProperty("user.home") + "/MNIST/images-idx1-ubyte";
    String labelsPath =  "file://" + System.getProperty("user.home") + "/MNIST/labels-idx1-ubyte";
    Map<String, String> params = new HashMap<>();
    params.put("imagesPath", imagesPath);
    params.put("labelsPath", labelsPath);
    params.put("recordsPerPartition", "400");
    params.put("maxRecords", "2000");

    DataFrame data = jsql.read().format(DefaultSource.class.getName())
            .options(params).load();

    System.out.println("\nLoaded Mnist dataframe:");
    data.show(100);

    DataFrame trainingData = data.sample(false, 0.8, 123);
    DataFrame testData = data.except(trainingData);

    StandardScaler scaler = new StandardScaler()
            .setInputCol("features")
            .setOutputCol("scaledFeatures");
    NeuralNetworkClassification classification = new NeuralNetworkClassification()
            .setFeaturesCol("scaledFeatures")
            .setEpochs(2)
            .setConf(getConfiguration());
    Pipeline pipeline = new Pipeline().setStages(new PipelineStage[]{
            scaler, classification});

    System.out.println("\nTraining...");
    PipelineModel model = pipeline.fit(trainingData);

    System.out.println("\nTesting...");
    DataFrame predictions = model.transform(testData);
    predictions.cache();

    System.out.println("\nTest Results:");
    predictions.show(100);

    Evaluation eval = new Evaluation(outputNum);
    Row[] rows = predictions.select("label","prediction").collect();
    for(int i = 0; i < rows.length; i++) {
        INDArray label = FeatureUtil.toOutcomeVector((int) rows[i].getDouble(0), outputNum);
        INDArray prediction = FeatureUtil.toOutcomeVector((int) rows[i].getDouble(1), outputNum);
        eval.eval(label, prediction);
    }

    System.out.println(eval.stats());
}
 
开发者ID:nitish11,项目名称:deeplearning4j-spark-ml-examples,代码行数:54,代码来源:JavaMnistClassification.java

示例5: main

import org.apache.spark.ml.PipelineModel; //导入方法依赖的package包/类
public static void main(String[] args) {
    SparkConf conf = new SparkConf().setMaster("local[*]")
            .setAppName("Cards Identification Pipeline (Java)");
    SparkContext jsc = new SparkContext(conf);
    SQLContext jsql = new SQLContext(jsc);

    String imagesPath =  "file://" + System.getProperty("user.home") + "/MNIST/images-idx1-ubyte";
    String labelsPath =  "file://" + System.getProperty("user.home") + "/MNIST/labels-idx1-ubyte";
    Map<String, String> params = new HashMap<>();
    params.put("imagesPath", imagesPath);
    params.put("labelsPath", labelsPath);
    params.put("recordsPerPartition", "400");
    params.put("maxRecords", "2000");

    DataFrame data = jsql.read().format(DefaultSource.class.getName())
            .options(params).load();

    System.out.println("\nLoaded Card Images dataframe:");
    data.show(100);

    DataFrame trainingData = data.sample(false, 0.8, 123);
    DataFrame testData = data.except(trainingData);

    StandardScaler scaler = new StandardScaler()
            .setInputCol("features")
            .setOutputCol("scaledFeatures");
    NeuralNetworkClassification classification = new NeuralNetworkClassification()
            .setFeaturesCol("scaledFeatures")
            .setEpochs(2)
            .setConf(getConfiguration());
    Pipeline pipeline = new Pipeline().setStages(new PipelineStage[]{
            scaler, classification});

    System.out.println("\nTraining...");
    PipelineModel model = pipeline.fit(trainingData);

    System.out.println("\nTesting...");
    DataFrame predictions = model.transform(testData);
    predictions.cache();

    System.out.println("\nTest Results:");
    predictions.show(100);

    Evaluation eval = new Evaluation(outputNum);
    Row[] rows = predictions.select("label","prediction").collect();
    for(int i = 0; i < rows.length; i++) {
        INDArray label = FeatureUtil.toOutcomeVector((int) rows[i].getDouble(0), outputNum);
        INDArray prediction = FeatureUtil.toOutcomeVector((int) rows[i].getDouble(1), outputNum);
        eval.eval(label, prediction);
    }

    System.out.println(eval.stats());
}
 
开发者ID:nitish11,项目名称:deeplearning4j-spark-ml-examples,代码行数:54,代码来源:JavaCardsIdentification.java


注:本文中的org.apache.spark.ml.PipelineModel.transform方法示例由纯净天空整理自Github/MSDocs等开源代码及文档管理平台,相关代码片段筛选自各路编程大神贡献的开源项目,源码版权归原作者所有,传播和使用请参考对应项目的License;未经允许,请勿转载。