Java JavaSparkContext.newAPIHadoopRDD方法代码示例

import org.apache.spark.api.java.JavaSparkContext; //导入方法依赖的package包/类
public LassoValidationSummary validateLassoAthenaFeatures(JavaSparkContext sc,
                                                          FeatureConstraint featureConstraint,
                                                          AthenaMLFeatureConfiguration athenaMLFeatureConfiguration,
                                                          LassoDetectionModel lassoDetectionModel,
                                                          Indexing indexing, Marking marking) {
    long start = System.nanoTime(); // <-- start

    JavaPairRDD<Object, BSONObject> mongoRDD;
    mongoRDD = sc.newAPIHadoopRDD(
            mongodbConfig,            // Configuration
            MongoInputFormat.class,   // InputFormat: read from a live cluster.
            Object.class,             // Key class
            BSONObject.class          // Value class
    );

    LassoDetectionAlgorithm lassoDetectionAlgorithm =
            (LassoDetectionAlgorithm) lassoDetectionModel.getDetectionAlgorithm();

    LassoValidationSummary lassoValidationSummary = new LassoValidationSummary();
    lassoValidationSummary.setLassoDetectionAlgorithm(lassoDetectionAlgorithm);
    LassoDistJob lassoDistJob = new LassoDistJob();

    lassoDistJob.validate(mongoRDD,
            athenaMLFeatureConfiguration,
            lassoDetectionModel,
            lassoValidationSummary);


    long end = System.nanoTime(); // <-- start
    long time = end - start;
    lassoValidationSummary.setValidationTime(time);

    return lassoValidationSummary;
}
 

import org.apache.spark.api.java.JavaSparkContext; //导入方法依赖的package包/类
public LinearRegressionValidationSummary validateLinearRegressionAthenaFeatures(JavaSparkContext sc,
                                                                                FeatureConstraint featureConstraint,
                                                                                AthenaMLFeatureConfiguration athenaMLFeatureConfiguration,
                                                                                LinearRegressionDetectionModel linearRegressionDetectionModel,
                                                                                Indexing indexing, Marking marking) {
    long start = System.nanoTime(); // <-- start

    JavaPairRDD<Object, BSONObject> mongoRDD;
    mongoRDD = sc.newAPIHadoopRDD(
            mongodbConfig,            // Configuration
            MongoInputFormat.class,   // InputFormat: read from a live cluster.
            Object.class,             // Key class
            BSONObject.class          // Value class
    );

    LinearRegressionDetectionAlgorithm linearRegressionDetectionAlgorithm =
            (LinearRegressionDetectionAlgorithm) linearRegressionDetectionModel.getDetectionAlgorithm();

    LinearRegressionValidationSummary linearRegressionValidationSummary =
            new LinearRegressionValidationSummary();
    linearRegressionValidationSummary.setLinearRegressionDetectionAlgorithm(linearRegressionDetectionAlgorithm);

    LinearRegressionDistJob linearRegressionDistJob = new LinearRegressionDistJob();

    linearRegressionDistJob.validate(mongoRDD,
            athenaMLFeatureConfiguration,
            linearRegressionDetectionModel,
            linearRegressionValidationSummary);


    long end = System.nanoTime(); // <-- start
    long time = end - start;
    linearRegressionValidationSummary.setValidationTime(time);


    return linearRegressionValidationSummary;
}
 

import org.apache.spark.api.java.JavaSparkContext; //导入方法依赖的package包/类
public SVMValidationSummary validateSVMAthenaFeatures(JavaSparkContext sc,
                                                      FeatureConstraint featureConstraint,
                                                      AthenaMLFeatureConfiguration athenaMLFeatureConfiguration,
                                                      SVMDetectionModel svmDetectionModel,
                                                      Indexing indexing, Marking marking) {
    long start = System.nanoTime(); // <-- start

    JavaPairRDD<Object, BSONObject> mongoRDD;
    mongoRDD = sc.newAPIHadoopRDD(
            mongodbConfig,            // Configuration
            MongoInputFormat.class,   // InputFormat: read from a live cluster.
            Object.class,             // Key class
            BSONObject.class          // Value class
    );

    SVMDetectionAlgorithm svmDetectionAlgorithm =
            (SVMDetectionAlgorithm) svmDetectionModel.getDetectionAlgorithm();

    SVMValidationSummary svmValidationSummary =
            new SVMValidationSummary(sc.sc(),
                    2, indexing, marking);

    SVMDistJob svmDistJob = new SVMDistJob();

    svmDistJob.validate(mongoRDD,
            athenaMLFeatureConfiguration,
            svmDetectionModel,
            svmValidationSummary);


    long end = System.nanoTime(); // <-- start
    long time = end - start;

    svmValidationSummary.setTotalValidationTime(time);
    return svmValidationSummary;
}
 

import org.apache.spark.api.java.JavaSparkContext; //导入方法依赖的package包/类
public GradientBoostedTreesValidationSummary validateGradientBoostedTreesAthenaFeatures(JavaSparkContext sc,
                                                                                        FeatureConstraint featureConstraint,
                                                                                        AthenaMLFeatureConfiguration athenaMLFeatureConfiguration,
                                                                                        GradientBoostedTreesDetectionModel gradientBoostedTreesDetectionModel,
                                                                                        Indexing indexing, Marking marking) {
    long start = System.nanoTime(); // <-- start

    JavaPairRDD<Object, BSONObject> mongoRDD;
    mongoRDD = sc.newAPIHadoopRDD(
            mongodbConfig,            // Configuration
            MongoInputFormat.class,   // InputFormat: read from a live cluster.
            Object.class,             // Key class
            BSONObject.class          // Value class
    );

    GradientBoostedTreesDetectionAlgorithm gradientBoostedTreesDetectionAlgorithm =
            (GradientBoostedTreesDetectionAlgorithm) gradientBoostedTreesDetectionModel.getDetectionAlgorithm();

    GradientBoostedTreesValidationSummary gradientBoostedTreesValidationSummary =
            new GradientBoostedTreesValidationSummary(sc.sc(),
                    gradientBoostedTreesDetectionAlgorithm.getNumClasses(), indexing, marking);

    GradientBoostedTreesDistJob gradientBoostedTreesDistJob = new GradientBoostedTreesDistJob();

    gradientBoostedTreesDistJob.validate(mongoRDD,
            athenaMLFeatureConfiguration,
            gradientBoostedTreesDetectionModel,
            gradientBoostedTreesValidationSummary);


    long end = System.nanoTime(); // <-- start
    long time = end - start;

    gradientBoostedTreesValidationSummary.setTotalValidationTime(time);
    return gradientBoostedTreesValidationSummary;
}
 

import org.apache.spark.api.java.JavaSparkContext; //导入方法依赖的package包/类
public RandomForestValidationSummary validateRandomForestAthenaFeatures(JavaSparkContext sc,
                                                                        FeatureConstraint featureConstraint,
                                                                        AthenaMLFeatureConfiguration athenaMLFeatureConfiguration,
                                                                        RandomForestDetectionModel randomForestDetectionModel,
                                                                        Indexing indexing, Marking marking) {
    long start = System.nanoTime(); // <-- start

    JavaPairRDD<Object, BSONObject> mongoRDD;
    mongoRDD = sc.newAPIHadoopRDD(
            mongodbConfig,            // Configuration
            MongoInputFormat.class,   // InputFormat: read from a live cluster.
            Object.class,             // Key class
            BSONObject.class          // Value class
    );

    RandomForestDetectionAlgorithm randomForestDetectionAlgorithm = (RandomForestDetectionAlgorithm) randomForestDetectionModel.getDetectionAlgorithm();

    RandomForestValidationSummary randomForestValidationSummary =
            new RandomForestValidationSummary(sc.sc(), randomForestDetectionAlgorithm.getNumClasses(), indexing, marking);

    RandomForestDistJob randomForestDistJob = new RandomForestDistJob();

    randomForestDistJob.validate(mongoRDD,
            athenaMLFeatureConfiguration,
            randomForestDetectionModel,
            randomForestValidationSummary);


    long end = System.nanoTime(); // <-- start
    long time = end - start;

    randomForestValidationSummary.setTotalValidationTime(time);
    return randomForestValidationSummary;
}
 

import org.apache.spark.api.java.JavaSparkContext; //导入方法依赖的package包/类
public NaiveBayesValidationSummary validateNaiveBayesAthenaFeatures(JavaSparkContext sc,
                                                                    FeatureConstraint featureConstraint,
                                                                    AthenaMLFeatureConfiguration athenaMLFeatureConfiguration,
                                                                    NaiveBayesDetectionModel naiveBayesDetectionModel,
                                                                    Indexing indexing, Marking marking) {
    long start = System.nanoTime(); // <-- start

    JavaPairRDD<Object, BSONObject> mongoRDD;
    mongoRDD = sc.newAPIHadoopRDD(
            mongodbConfig,            // Configuration
            MongoInputFormat.class,   // InputFormat: read from a live cluster.
            Object.class,             // Key class
            BSONObject.class          // Value class
    );

    NaiveBayesDetectionAlgorithm naiveBayesDetectionAlgorithm = (NaiveBayesDetectionAlgorithm) naiveBayesDetectionModel.getDetectionAlgorithm();

    NaiveBayesValidationSummary naiveBayesValidationSummary =
            new NaiveBayesValidationSummary(sc.sc(), naiveBayesDetectionAlgorithm.getNumClasses(), indexing, marking);

    NaiveBayesDistJob naiveBayesDistJob = new NaiveBayesDistJob();


    naiveBayesDistJob.validate(mongoRDD,
            athenaMLFeatureConfiguration,
            naiveBayesDetectionModel,
            naiveBayesValidationSummary);


    long end = System.nanoTime(); // <-- start
    long time = end - start;

    naiveBayesValidationSummary.setTotalValidationTime(time);
    return naiveBayesValidationSummary;
}
 

import org.apache.spark.api.java.JavaSparkContext; //导入方法依赖的package包/类
public DecisionTreeValidationSummary validateDecisionTreeAthenaFeatures(JavaSparkContext sc,
                                                                        FeatureConstraint featureConstraint,
                                                                        AthenaMLFeatureConfiguration athenaMLFeatureConfiguration,
                                                                        DecisionTreeDetectionModel decisionTreeDetectionModel,
                                                                        Indexing indexing, Marking marking) {
    long start = System.nanoTime(); // <-- start

    JavaPairRDD<Object, BSONObject> mongoRDD;
    mongoRDD = sc.newAPIHadoopRDD(
            mongodbConfig,            // Configuration
            MongoInputFormat.class,   // InputFormat: read from a live cluster.
            Object.class,             // Key class
            BSONObject.class          // Value class
    );

    DecisionTreeDetectionAlgorithm decisionTreeDetectionAlgorithm = (DecisionTreeDetectionAlgorithm) decisionTreeDetectionModel.getDetectionAlgorithm();

    DecisionTreeValidationSummary decisionTreeValidationSummary =
            new DecisionTreeValidationSummary(sc.sc(), decisionTreeDetectionAlgorithm.getNumClasses(), indexing, marking);

    DecisionTreeDistJob decisionTreeDistJob = new DecisionTreeDistJob();

    decisionTreeDistJob.validate(mongoRDD,
            athenaMLFeatureConfiguration,
            decisionTreeDetectionModel,
            decisionTreeValidationSummary);


    long end = System.nanoTime(); // <-- start
    long time = end - start;

    decisionTreeValidationSummary.setTotalValidationTime(time);
    return decisionTreeValidationSummary;
}
 

import org.apache.spark.api.java.JavaSparkContext; //导入方法依赖的package包/类
public GaussianMixtureValidationSummary validateGaussianMixtureAthenaFeatures(JavaSparkContext sc,
                                                                              FeatureConstraint featureConstraint,
                                                                              AthenaMLFeatureConfiguration athenaMLFeatureConfiguration,
                                                                              GaussianMixtureDetectionModel gaussianMixtureDetectionModel,
                                                                              Indexing indexing,
                                                                              Marking marking) {
    long start = System.nanoTime(); // <-- start

    JavaPairRDD<Object, BSONObject> mongoRDD;
    mongoRDD = sc.newAPIHadoopRDD(
            mongodbConfig,            // Configuration
            MongoInputFormat.class,   // InputFormat: read from a live cluster.
            Object.class,             // Key class
            BSONObject.class          // Value class
    );

    GaussianMixtureDetectionAlgorithm gaussianMixtureDetectionAlgorithm = (GaussianMixtureDetectionAlgorithm) gaussianMixtureDetectionModel.getDetectionAlgorithm();
    GaussianMixtureValidationSummary gaussianMixtureValidationSummary =
            new GaussianMixtureValidationSummary(sc.sc(), gaussianMixtureDetectionAlgorithm.getK(), indexing, marking);


    GaussianMixtureDistJob gaussianMixtureDistJob = new GaussianMixtureDistJob();

    gaussianMixtureDistJob.validate(mongoRDD,
            athenaMLFeatureConfiguration,
            gaussianMixtureDetectionModel,
            gaussianMixtureValidationSummary);


    long end = System.nanoTime(); // <-- start
    long time = end - start;

    gaussianMixtureValidationSummary.setTotalValidationTime(time);
    return gaussianMixtureValidationSummary;
}
 

import org.apache.spark.api.java.JavaSparkContext; //导入方法依赖的package包/类
public KmeansValidationSummary validateKMeansAthenaFeatures(JavaSparkContext sc,
                                                            FeatureConstraint featureConstraint,
                                                            AthenaMLFeatureConfiguration athenaMLFeatureConfiguration,
                                                            KMeansDetectionModel kMeansDetectionModel,
                                                            Indexing indexing,
                                                            Marking marking) {
    long start = System.nanoTime(); // <-- start

    JavaPairRDD<Object, BSONObject> mongoRDD;
    mongoRDD = sc.newAPIHadoopRDD(
            mongodbConfig,            // Configuration
            MongoInputFormat.class,   // InputFormat: read from a live cluster.
            Object.class,             // Key class
            BSONObject.class          // Value class
    );

    KMeansDetectionAlgorithm kMeansDetectionAlgorithm = (KMeansDetectionAlgorithm) kMeansDetectionModel.getDetectionAlgorithm();

    KmeansValidationSummary kmeansValidationSummary =
            new KmeansValidationSummary(sc.sc(), kMeansDetectionAlgorithm.getK(), indexing, marking);

    KMeansDistJob KMeansDistJob = new KMeansDistJob();

    KMeansDistJob.validate(mongoRDD,
            athenaMLFeatureConfiguration,
            kMeansDetectionModel,
            kmeansValidationSummary);
    long end = System.nanoTime(); // <-- start
    long time = end - start;
    kmeansValidationSummary.setTotalValidationTime(time);
    return kmeansValidationSummary;
}
 

import org.apache.spark.api.java.JavaSparkContext; //导入方法依赖的package包/类
public GaussianMixtureDetectionModel generateGaussianMixtureAthenaDetectionModel(JavaSparkContext sc,
                                                                                 FeatureConstraint featureConstraint,
                                                                                 AthenaMLFeatureConfiguration athenaMLFeatureConfiguration,
                                                                                 DetectionAlgorithm detectionAlgorithm,
                                                                                 Indexing indexing,
                                                                                 Marking marking) {
    GaussianMixtureModelSummary gaussianMixtureModelSummary = new GaussianMixtureModelSummary(
            sc.sc(), indexing, marking);
    long start = System.nanoTime(); // <-- start

    GaussianMixtureDetectionAlgorithm gaussianMixtureDetectionAlgorithm = (GaussianMixtureDetectionAlgorithm) detectionAlgorithm;

    GaussianMixtureDetectionModel gaussianMixtureDetectionModel = new GaussianMixtureDetectionModel();
    gaussianMixtureDetectionModel.setGaussianMixtureDetectionAlgorithm(gaussianMixtureDetectionAlgorithm);
    gaussianMixtureModelSummary.setGaussianMixtureDetectionAlgorithm(gaussianMixtureDetectionAlgorithm);
    gaussianMixtureDetectionModel.setFeatureConstraint(featureConstraint);
    gaussianMixtureDetectionModel.setAthenaMLFeatureConfiguration(athenaMLFeatureConfiguration);
    gaussianMixtureDetectionModel.setIndexing(indexing);
    gaussianMixtureDetectionModel.setMarking(marking);

    JavaPairRDD<Object, BSONObject> mongoRDD;
    mongoRDD = sc.newAPIHadoopRDD(
            mongodbConfig,            // Configuration
            MongoInputFormat.class,   // InputFormat: read from a live cluster.
            Object.class,             // Key class
            BSONObject.class          // Value class
    );

    GaussianMixtureDistJob gaussianMixtureDistJob = new GaussianMixtureDistJob();

    GaussianMixtureModel gaussianMixtureModel = gaussianMixtureDistJob.generateGaussianMixtureWithPreprocessing(mongoRDD,
            athenaMLFeatureConfiguration, gaussianMixtureDetectionAlgorithm, gaussianMixtureModelSummary);


    gaussianMixtureDetectionModel.setkGaussianMixtureModel(gaussianMixtureModel);

    long end = System.nanoTime(); // <-- start
    long time = end - start;
    gaussianMixtureModelSummary.setTotalLearningTime(time);
    gaussianMixtureDetectionModel.setClusterModelSummary(gaussianMixtureModelSummary);
    gaussianMixtureModelSummary.setGaussianMixtureModel(gaussianMixtureModel);
    return gaussianMixtureDetectionModel;
}
 

import org.apache.spark.api.java.JavaSparkContext; //导入方法依赖的package包/类
public KMeansDetectionModel generateKMeansAthenaDetectionModel(JavaSparkContext sc,
                                                               FeatureConstraint featureConstraint,
                                                               AthenaMLFeatureConfiguration athenaMLFeatureConfiguration,
                                                               DetectionAlgorithm detectionAlgorithm,
                                                               Indexing indexing,
                                                               Marking marking) {
    KmeansModelSummary kmeansModelSummary = new KmeansModelSummary(sc.sc(), indexing, marking);

    long start = System.nanoTime(); // <-- start

    KMeansDetectionAlgorithm kMeansDetectionAlgorithm = (KMeansDetectionAlgorithm) detectionAlgorithm;
    KMeansDetectionModel kMeansDetectionModel = new KMeansDetectionModel();
    kMeansDetectionModel.setkMeansDetectionAlgorithm(kMeansDetectionAlgorithm);
    kmeansModelSummary.setkMeansDetectionAlgorithm(kMeansDetectionAlgorithm);
    kMeansDetectionModel.setFeatureConstraint(featureConstraint);
    kMeansDetectionModel.setAthenaMLFeatureConfiguration(athenaMLFeatureConfiguration);
    kMeansDetectionModel.setIndexing(indexing);
    kMeansDetectionModel.setMarking(marking);

    JavaPairRDD<Object, BSONObject> mongoRDD;
    mongoRDD = sc.newAPIHadoopRDD(
            mongodbConfig,            // Configuration
            MongoInputFormat.class,   // InputFormat: read from a live cluster.
            Object.class,             // Key class
            BSONObject.class          // Value class
    );

    KMeansDistJob KMeansDistJob = new KMeansDistJob();


    KMeansModel kMeansModel = KMeansDistJob.generateKmeansWithPreprocessing(mongoRDD,
            athenaMLFeatureConfiguration, kMeansDetectionAlgorithm, kmeansModelSummary);


    kMeansDetectionModel.setkMeansModel(kMeansModel);
    long end = System.nanoTime(); // <-- start
    long time = end - start;
    kmeansModelSummary.setTotalLearningTime(time);
    kMeansDetectionModel.setClusterModelSummary(kmeansModelSummary);
    return kMeansDetectionModel;
}
 

import org.apache.spark.api.java.JavaSparkContext; //导入方法依赖的package包/类
public LassoDetectionModel generateLassoAthenaDetectionModel(JavaSparkContext sc,
                                                             FeatureConstraint featureConstraint,
                                                             AthenaMLFeatureConfiguration athenaMLFeatureConfiguration,
                                                             DetectionAlgorithm detectionAlgorithm,
                                                             Indexing indexing,
                                                             Marking marking) {
    LassoModelSummary lassoModelSummary = new LassoModelSummary(
            sc.sc(), indexing, marking);

    long start = System.nanoTime(); // <-- start

    LassoDetectionAlgorithm lassoDetectionAlgorithm = (LassoDetectionAlgorithm) detectionAlgorithm;

    LassoDetectionModel lassoDetectionModel = new LassoDetectionModel();

    lassoDetectionModel.setLassoDetectionAlgorithm(lassoDetectionAlgorithm);
    lassoModelSummary.setLassoDetectionAlgorithm(lassoDetectionAlgorithm);
    lassoDetectionModel.setFeatureConstraint(featureConstraint);
    lassoDetectionModel.setAthenaMLFeatureConfiguration(athenaMLFeatureConfiguration);
    lassoDetectionModel.setIndexing(indexing);
    lassoDetectionModel.setMarking(marking);

    JavaPairRDD<Object, BSONObject> mongoRDD;
    mongoRDD = sc.newAPIHadoopRDD(
            mongodbConfig,            // Configuration
            MongoInputFormat.class,   // InputFormat: read from a live cluster.
            Object.class,             // Key class
            BSONObject.class          // Value class
    );

    LassoDistJob lassoDistJob = new LassoDistJob();

    LassoModel lassoModel = lassoDistJob.generateDecisionTreeWithPreprocessing(mongoRDD,
            athenaMLFeatureConfiguration, lassoDetectionAlgorithm, marking, lassoModelSummary);


    lassoDetectionModel.setModel(lassoModel);
    long end = System.nanoTime(); // <-- start
    long time = end - start;
    lassoModelSummary.setTotalLearningTime(time);
    lassoDetectionModel.setClassificationModelSummary(lassoModelSummary);

    return lassoDetectionModel;
}
 

import org.apache.spark.api.java.JavaSparkContext; //导入方法依赖的package包/类
public RidgeRegressionDetectionModel generateRidgeRegressionAthenaDetectionModel(JavaSparkContext sc,
                                                                                 FeatureConstraint featureConstraint,
                                                                                 AthenaMLFeatureConfiguration athenaMLFeatureConfiguration,
                                                                                 DetectionAlgorithm detectionAlgorithm,
                                                                                 Indexing indexing,
                                                                                 Marking marking) {
    RidgeRegressionModelSummary ridgeRegressionModelSummary = new RidgeRegressionModelSummary(
            sc.sc(), indexing, marking);

    long start = System.nanoTime(); // <-- start

    RidgeRegressionDetectionAlgorithm ridgeRegressionDetectionAlgorithm = (RidgeRegressionDetectionAlgorithm) detectionAlgorithm;

    RidgeRegressionDetectionModel ridgeRegressionDetectionModel = new RidgeRegressionDetectionModel();

    ridgeRegressionDetectionModel.setRidgeRegressionDetectionAlgorithm(ridgeRegressionDetectionAlgorithm);
    ridgeRegressionModelSummary.setRidgeRegressionDetectionAlgorithm(ridgeRegressionDetectionAlgorithm);
    ridgeRegressionDetectionModel.setFeatureConstraint(featureConstraint);
    ridgeRegressionDetectionModel.setAthenaMLFeatureConfiguration(athenaMLFeatureConfiguration);
    ridgeRegressionDetectionModel.setIndexing(indexing);
    ridgeRegressionDetectionModel.setMarking(marking);

    JavaPairRDD<Object, BSONObject> mongoRDD;
    mongoRDD = sc.newAPIHadoopRDD(
            mongodbConfig,            // Configuration
            MongoInputFormat.class,   // InputFormat: read from a live cluster.
            Object.class,             // Key class
            BSONObject.class          // Value class
    );

    RidgeRegressionDistJob ridgeRegressionDistJob = new RidgeRegressionDistJob();

    RidgeRegressionModel ridgeRegressionModel = ridgeRegressionDistJob.generateDecisionTreeWithPreprocessing(mongoRDD,
            athenaMLFeatureConfiguration, ridgeRegressionDetectionAlgorithm, marking, ridgeRegressionModelSummary);


    ridgeRegressionDetectionModel.setModel(ridgeRegressionModel);
    long end = System.nanoTime(); // <-- start
    long time = end - start;
    ridgeRegressionModelSummary.setTotalLearningTime(time);
    ridgeRegressionDetectionModel.setClassificationModelSummary(ridgeRegressionModelSummary);

    return ridgeRegressionDetectionModel;
}
 

import org.apache.spark.api.java.JavaSparkContext; //导入方法依赖的package包/类
public LinearRegressionDetectionModel generateLinearRegressionAthenaDetectionModel(JavaSparkContext sc,
                                                                                   FeatureConstraint featureConstraint,
                                                                                   AthenaMLFeatureConfiguration athenaMLFeatureConfiguration,
                                                                                   DetectionAlgorithm detectionAlgorithm,
                                                                                   Indexing indexing,
                                                                                   Marking marking) {
    LinearRegressionModelSummary linearRegressionModelSummary = new LinearRegressionModelSummary(
            sc.sc(), indexing, marking);

    long start = System.nanoTime(); // <-- start

    LinearRegressionDetectionAlgorithm linearRegressionDetectionAlgorithm = (LinearRegressionDetectionAlgorithm) detectionAlgorithm;

    LinearRegressionDetectionModel linearRegressionDetectionModel = new LinearRegressionDetectionModel();

    linearRegressionDetectionModel.setLinearRegressionDetectionAlgorithm(linearRegressionDetectionAlgorithm);
    linearRegressionModelSummary.setLinearRegressionDetectionAlgorithm(linearRegressionDetectionAlgorithm);
    linearRegressionDetectionModel.setFeatureConstraint(featureConstraint);
    linearRegressionDetectionModel.setAthenaMLFeatureConfiguration(athenaMLFeatureConfiguration);
    linearRegressionDetectionModel.setIndexing(indexing);
    linearRegressionDetectionModel.setMarking(marking);

    JavaPairRDD<Object, BSONObject> mongoRDD;
    mongoRDD = sc.newAPIHadoopRDD(
            mongodbConfig,            // Configuration
            MongoInputFormat.class,   // InputFormat: read from a live cluster.
            Object.class,             // Key class
            BSONObject.class          // Value class
    );

    LinearRegressionDistJob linearRegressionDistJob = new LinearRegressionDistJob();

    LinearRegressionModel linearRegressionModel = linearRegressionDistJob.generateDecisionTreeWithPreprocessing(mongoRDD,
            athenaMLFeatureConfiguration, linearRegressionDetectionAlgorithm, marking, linearRegressionModelSummary);


    linearRegressionDetectionModel.setModel(linearRegressionModel);
    long end = System.nanoTime(); // <-- start
    long time = end - start;
    linearRegressionModelSummary.setTotalLearningTime(time);
    linearRegressionDetectionModel.setClassificationModelSummary(linearRegressionModelSummary);

    return linearRegressionDetectionModel;
}
 

import org.apache.spark.api.java.JavaSparkContext; //导入方法依赖的package包/类
public LogisticRegressionDetectionModel generateLogisticRegressionAthenaDetectionModel(JavaSparkContext sc,
                                                                                       FeatureConstraint featureConstraint,
                                                                                       AthenaMLFeatureConfiguration athenaMLFeatureConfiguration,
                                                                                       DetectionAlgorithm detectionAlgorithm,
                                                                                       Indexing indexing,
                                                                                       Marking marking) {
    LogisticRegressionModelSummary logisticRegressionModelSummary = new LogisticRegressionModelSummary(
            sc.sc(), indexing, marking);

    long start = System.nanoTime(); // <-- start

    LogisticRegressionDetectionAlgorithm logisticRegressionDetectionAlgorithm = (LogisticRegressionDetectionAlgorithm) detectionAlgorithm;

    LogisticRegressionDetectionModel logisticRegressionDetectionModel = new LogisticRegressionDetectionModel();

    logisticRegressionDetectionModel.setLogisticRegressionDetectionAlgorithm(logisticRegressionDetectionAlgorithm);
    logisticRegressionModelSummary.setLogisticRegressionDetectionAlgorithm(logisticRegressionDetectionAlgorithm);
    logisticRegressionDetectionModel.setFeatureConstraint(featureConstraint);
    logisticRegressionDetectionModel.setAthenaMLFeatureConfiguration(athenaMLFeatureConfiguration);
    logisticRegressionDetectionModel.setIndexing(indexing);
    logisticRegressionDetectionModel.setMarking(marking);

    JavaPairRDD<Object, BSONObject> mongoRDD;
    mongoRDD = sc.newAPIHadoopRDD(
            mongodbConfig,            // Configuration
            MongoInputFormat.class,   // InputFormat: read from a live cluster.
            Object.class,             // Key class
            BSONObject.class          // Value class
    );

    LogisticRegressionDistJob logisticRegressionDistJob = new LogisticRegressionDistJob();

    LogisticRegressionModel logisticRegressionModel = logisticRegressionDistJob.generateDecisionTreeWithPreprocessing(mongoRDD,
            athenaMLFeatureConfiguration, logisticRegressionDetectionAlgorithm, marking, logisticRegressionModelSummary);


    logisticRegressionDetectionModel.setModel(logisticRegressionModel);
    long end = System.nanoTime(); // <-- start
    long time = end - start;
    logisticRegressionModelSummary.setTotalLearningTime(time);
    logisticRegressionDetectionModel.setClassificationModelSummary(logisticRegressionModelSummary);

    return logisticRegressionDetectionModel;
}