Scala DataSet类代码示例

//设置package包名称以及导入依赖的类
package fastml4j.classification

import org.nd4s.Implicits._
import org.nd4j.linalg.api.ndarray.INDArray
import org.nd4j.linalg.factory.Nd4j
import fastml4j.optimizer._
import fastml4j.losses._
import org.nd4j.linalg.dataset.DataSet

class SVM(val lambdaL2: Double,
  val alpha: Double = 0.01,
  val maxIterations: Int = 1000,
  val stohasticBatchSize: Int = 100,
  val optimizerType: String = "PegasosSGD",
  val eps: Double = 1e-6) extends ClassificationModel {

  var weights: INDArray = Nd4j.zeros(1)
  var losses: Seq[Double] = Seq[Double]()


  def fit(dataSet: DataSet, initWeights: Option[INDArray] = None) = {

    val optimizer: Optimizer = optimizerType match {
      case "GradientDescent" => new GradientDescent(maxIterations, alpha, eps)
    //  case "GradientDescentDecreasingLearningRate" => new GradientDescentDecreasingLearningRate(maxIterations, alpha, eps)
      case "PegasosSGD" => new PegasosSGD(maxIterations, alpha, eps)
      case _ => throw new Exception("Optimizer %s is not supported".format(optimizerType))
    }

    val (weightsOut, lossesOut) = optimizer.optimize(
      new HingeLoss(lambdaL2),
      initWeights = initWeights.getOrElse(Nd4j.zeros(dataSet.numInputs)),
      dataSet)

    weights = weightsOut
    losses = lossesOut
  }

  def predictClass(inputVector: INDArray): Double = {
    val sign = math.signum((inputVector dot weights.T).sumT[Double])
    if( sign != 0 ) sign else 1.0
  }

  def predict(inputVector:  INDArray): Double = {
    (inputVector dot weights).sumT[Double]
  }


} 

//设置package包名称以及导入依赖的类
package fastml4j.losses

import org.nd4s.Implicits._
import org.nd4j.linalg.api.ndarray.INDArray
import org.nd4j.linalg.dataset.DataSet
import org.nd4j.linalg.factory.Nd4j


abstract class Loss{
  protected var lambdaL2: Double = 0
  protected var lambdaL1: Double = 0

  def loss(weights: INDArray, dataSet: DataSet): Double
  def gradient(weights: INDArray,dataSet: DataSet): INDArray

  // More about gradient checking:  http://cs231n.github.io/neural-networks-3/
  def numericGradient(weights: INDArray, dataSet: DataSet, eps: Double = 1e-6): INDArray =
    (0 to (weights.columns() - 1)).map {
      i =>
        val oldWeights = weights.dup.put(0, i, weights.get(0,i) - eps)
        val newWeights = weights.dup.put(0, i, weights.get(0,i) + eps)

        (loss(newWeights, dataSet) - loss(oldWeights, dataSet)) / 2.0 / eps }
      .toNDArray

} 

//设置package包名称以及导入依赖的类
package fastml4j.regression

import fastml4j.losses.OLSLoss
import fastml4j.optimizer.GradientDescent
import org.nd4j.linalg.api.ndarray.INDArray
import org.nd4j.linalg.dataset.DataSet
import org.nd4j.linalg.factory.Nd4j
import org.nd4s.Implicits._


class LinearRegression(val lambdaL2: Double,
  val alpha: Double = 0.01,
  val maxIterations: Int = 1000,
  val stohasticBatchSize: Int = 100,
  val optimizerType: String = "GradientDescent",
  val eps: Double = 1e-6) {

  var weights: INDArray = Nd4j.zeros(1)
  var losses: Seq[Double] = Seq[Double]()

  def fit(dataSet: DataSet, initWeights: Option[INDArray] = None): Unit = {

    val optimizer = optimizerType match {
      case "GradientDescent" => new GradientDescent(maxIterations, alpha, eps)
      case _ => throw new Exception("Optimizer %s is not supported".format(optimizerType))
    }

    val (weightsOut, lossesOut) = optimizer.optimize(new OLSLoss(lambdaL2),
      initWeights.getOrElse(Nd4j.zeros(dataSet.numExamples)),
      dataSet)
    weights = weightsOut
    losses = lossesOut
  }

  def predict(inputVector: INDArray): Double = {
    (inputVector dot weights).sumT[Double]
  }


} 

//设置package包名称以及导入依赖的类
package fastml4j.optimizer

import fastml4j.losses.Loss
import org.nd4s.Implicits._
import org.nd4j.linalg.api.ndarray.INDArray
import org.nd4j.linalg.dataset.DataSet
import org.nd4j.linalg.factory.Nd4j

import scala.annotation.tailrec


class PegasosSGD(
  val maxIterations: Int,
  val lambda: Double,
  val eps: Double = 1e-6,
  val batchSize: Int = 100,
  val withReplacement: Boolean = true) extends Optimizer {

  //http://ttic.uchicago.edu/~nati/Publications/PegasosMPB.pdf
  def optimize(loss: Loss, initWeights: INDArray, dataSet: DataSet): (INDArray, Seq[Double]) = {

    @tailrec
    def helperOptimizer( prevWeights:INDArray, losses: Seq[Double], batch: Int): (INDArray, Seq[Double]) = {
      val sampleDataSet = dataSet.sample(batchSize, withReplacement)
      val eta = 1.0 / lambda / batch
      val weights = prevWeights * (1 - lambda * eta)  -
        loss.gradient(prevWeights, sampleDataSet) * eta

      val currentLoss = loss.loss(weights, sampleDataSet)

      if( losses.size > 0 && ((math.abs(currentLoss - losses.last) < eps) || losses.size >= maxIterations))
        (weights, losses :+ currentLoss)
      else
        helperOptimizer(weights, losses :+ currentLoss, batch + 1)}

    helperOptimizer(initWeights, Seq[Double](), 1)

  }


} 

//设置package包名称以及导入依赖的类
package fastml4j.regression

import org.scalatest._
import org.nd4s.Implicits._
import org.nd4j.linalg.api.ndarray.INDArray
import org.nd4j.linalg.dataset.DataSet
import org.scalatest.Matchers._
import org.nd4j.linalg.factory.Nd4j


class LinearRegressionSuite extends FunSuite with BeforeAndAfter {

  test("Simple test") {
    val coef1 = 1.0
    val coef2 = 0.5

    val x = (0 to   500).map { x => Array(x.toDouble,1.0) }.toArray
    val y = x.map {case Array(x1, x2) => x1 * coef1 + x2 * coef2 + math.random/10 }

    val lr = new LinearRegression(lambdaL2 = 0.0, alpha = 0.000001, eps = 1e-4, maxIterations = 2000)
    lr.fit(new DataSet(x.toNDArray, y.toNDArray), Option(Nd4j.zeros(2)) )

    val weights = lr.weights
    assert(weights.getDouble(0,0) === coef1 +- 1)
    assert(weights.getDouble(0,1) === coef2 +- 1)
  }

} 

//设置package包名称以及导入依赖的类
package com.cloudera.datascience.dl4j.cnn.examples.caltech256

import java.io.File

import com.cloudera.datascience.dl4j.cnn.Utils
import org.apache.spark.sql.{Row, SparkSession}
import org.deeplearning4j.util.ModelSerializer
import org.nd4j.linalg.dataset.DataSet
import org.nd4j.linalg.factory.Nd4j
import scopt.OptionParser

object ScoreSparkModel {

  private[this] case class Params (modelPath: String = null, dataPath: String = null)

  private[this] object Params {
    def parseArgs(args: Array[String]): Params = {
      val params = new OptionParser[Params]("train an existing model") {
        opt[String]("test")
          .text("the path of the test data")
          .action((x, c) => c.copy(dataPath = x))
        opt[String]("model")
          .text("location of the model")
          .action((x, c) => c.copy(modelPath = x))
      }.parse(args, Params()).get
      require(params.modelPath != null && params.dataPath != null,
        "You must supply the data path and a model path.")
      params
    }
  }

  def main(args: Array[String]): Unit = {
    val param = Params.parseArgs(args)
    val spark = SparkSession.builder().appName("score a model").getOrCreate()
    try {
      val sc = spark.sparkContext
      sc.hadoopConfiguration.set("mapreduce.input.fileinputformat.input.dir.recursive", "true")
      val restorePath = new File(param.modelPath)
      val restored = ModelSerializer.restoreComputationGraph(restorePath)
      val testRDD = spark.read.parquet(param.dataPath)
        .rdd
        .map { case Row(f: Array[Byte], l: Array[Byte]) =>
          new DataSet(Nd4j.fromByteArray(f), Nd4j.fromByteArray(l))
        }
      val eval = Utils.evaluate(restored, testRDD, 16)
      println(Utils.prettyPrintEvaluationStats(eval))

    } finally {
      spark.stop()
    }
  }
}