Scala DenseMatrix类代码示例

//设置package包名称以及导入依赖的类
package breeze

import breeze.linalg.{DenseMatrix, DenseVector}
import org.apache.spark.{SparkConf, SparkContext}
import org.slf4j.LoggerFactory
import spark.RecommendationExample.getClass

/**
  * Created by I311352 on 4/5/2017.
  */
object rddvector extends App {
  val LOG = LoggerFactory.getLogger(getClass)

  val conf = new SparkConf().setAppName("vector").setMaster("local[2]")
  val sc = new SparkContext(conf)
  val data = sc.textFile("data/testdata.txt")
  println(data.take(10).toList)

  val vectorRDD = data.map(value => {
    val columns = value.split(",").map(value => value.toDouble)
    new DenseVector(columns)
  })

  println(vectorRDD.take(100).toList)

  // multiply each row by a constant vector
  val constant = 5.0
  val broadcastConstant = sc.broadcast(constant)
  val scaledRDD = vectorRDD.map(row => {
    row :* broadcastConstant.value
  })

  println(scaledRDD.take(10).toList)

  val scaledRDDByPartition = vectorRDD.glom().map((value:Array[DenseVector[Double]]) => {
    val arrayValues = value.map(denseVector => denseVector.data).flatten
    val denseMatrix = new DenseMatrix[Double](value.length,value(0).length,arrayValues)
    denseMatrix :*= broadcastConstant.value
    denseMatrix.toDenseVector
  })

  println(scaledRDDByPartition.take(10).toList)
} 

//设置package包名称以及导入依赖的类
package util

import breeze.linalg.{DenseMatrix, DenseVector, sum}
import regression.Regressor


object Evaluation {
  def confusion(lr: Regressor, data: Iterable[(DenseVector[Double], Double)]): DenseMatrix[Double] = {
    val confusion = DenseMatrix.zeros[Double](2, 2)

    data.map { case (x, y) =>
      (y.toInt, if (lr.predict(x) > 0.5) 1 else 0)
    } foreach { case (truth, predicted) =>
      confusion(truth, predicted) += 1.0
    }

    confusion
  }

  def printConfusionMtx(confusion: DenseMatrix[Double]): Unit = {
    val negatives = confusion(0, 0) + confusion(0, 1)
    val positives = confusion(1, 0) + confusion(1, 1)
    val total = sum(confusion)

    val falseNegatives = confusion(1, 0)
    val falsePositives = confusion(0, 1)
    val accuracy = (confusion(0, 0) + confusion(1, 1)) / total

    println("============= Stats =============\n")

    println(f"Positive examples: $positives%1.0f")
    println(f"Negative examples: $negatives%1.0f")
    println(f"Total: $total%1.0f")
    println(f"Pos/Neg ratio: ${positives/negatives}%1.2f")

    println("\n============= Results =============\n")

    println("Confusion Matrix:")
    println(confusion)
    println(f"Accuracy: ${accuracy * 100}%2.2f%%")
    println(f"False positives: ${falsePositives * 100 / negatives}%2.2f%%")
    println(f"False negatives: ${falseNegatives * 100 / positives}%2.2f%%")
  }
} 

//设置package包名称以及导入依赖的类
package AccurateML.lsh

import breeze.linalg.DenseMatrix
import org.apache.spark.mllib.linalg
import org.apache.spark.mllib.linalg.{Vector, Vectors}
import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}
import AccurateML.blas.ZFBLAS


class ZFLSH(
             n: Int,
             m: Int) {
  val normal01 = breeze.stats.distributions.Gaussian(0, 1)
  val nmat = DenseMatrix.rand(m, n, normal01)

  def hashVector(vector: linalg.Vector): String = {

    val r = new Array[Int](n)
    for (i <- 0 until n) {
      val mc = nmat(::, (i))
      val ans = ZFBLAS.dot(vector, Vectors.dense(mc.toArray))
      if (ans > 0)
        r(i) = 1
    }
    r.mkString("")
  }

  def main(args: Array[String]) {
    val conf = new SparkConf().setAppName("test lsh")
    val sc = new SparkContext(conf)
    val numBits = 4
    val numFeatures = 1000
    val lsh = new ZFLSH(numBits, numFeatures)
    val data: RDD[Vector] = sc.objectFile("") //eg, the first element in data is dfirst=Vector(1.0,2.0,...,1000.0)
    val mapData:RDD[(String,Vector)]=data.map(vec=>(lsh.hashVector(vec),vec))
    //eg,the first element in mapData mdfirst=Tuple2("1010",Vector(1.0,2.0,...,100.0))
    //"1010" is the sketch of dfirst=Vector(1.0,2.0,...,1000.0)
    //the instances with the same sketch will belong to the same cluster

  }

} 

//设置package包名称以及导入依赖的类
package linalg.matrix

import breeze.linalg.{DenseMatrix, _}
import breeze.numerics._



object BreezeMatrixOperations {

  def main(args: Array[String]) {
    val a = DenseMatrix((1,2),(3,4))
    val b = DenseMatrix((2,2),(2,2))
    val c = a + b
    println("a: \n" + a)
    println("b: \n" + b)
    println("a + b : \n" + c)
    val d = a*b
    println("Dot product a*b : \n" + d)

    val e = a :+= 1
    println("Inplace Addition : a :+= 1\n" + e)

    //a:*= 2.0
    //println("Inplace Multiplication : a :*= 2.0\n" + a :*=2.0)

    val f = a :< b
    println("a :< b \n" + f)
    val g = DenseMatrix((1.1, 1.2), (3.9, 3.5))
    println("g: \n" + g)
    val gCeil =ceil(g)
    println("ceil(g)\n " + gCeil)

    val gFloor =floor(g)
    println("floor(g)\n" + gFloor)

    val sumA = sum(a)
    println("sum(a):\n" + sumA)
    println("a.max:\n" + a.max)
    println("argmax(a):\n" + argmax(a))

  }

} 

//设置package包名称以及导入依赖的类
package io.flatmap.ml.som

import breeze.numerics.closeTo
import breeze.linalg.DenseMatrix
import io.flatmap.ml.som.SelfOrganizingMap.Shape
import org.apache.spark.mllib.linalg.DenseVector
import org.apache.spark.mllib.random.RandomRDDs
import org.scalatest._
import util.{FakeDecayFunction, FakeMetrics, FakeNeighborhoodKernel, TestSparkContext}

class SelfOrganizingMapSpec extends FlatSpec with Matchers with BeforeAndAfterEach with TestSparkContext {

  def SOM(width: Int, height: Int) =
    new SelfOrganizingMap with FakeNeighborhoodKernel with FakeDecayFunction with FakeMetrics {
      override val shape: Shape = (width, height)
      override val learningRate: Double = 0.1
      override val sigma: Double = 0.2
    }

  "instantiation" should "create a SOM with codebook of zeros" in {
    val som = SOM(6, 6)
    som.codeBook should === (DenseMatrix.fill[Array[Double]](6, 6)(Array.emptyDoubleArray))
  }

  "initialize" should "copy random data points from RDD into codebook" in {
    val data = RandomRDDs.normalVectorRDD(sparkSession.sparkContext, numRows = 512L, numCols = 3)
    val som = SOM(6, 6)
    som.initialize(data).codeBook should !== (DenseMatrix.fill[Array[Double]](6, 6)(Array.emptyDoubleArray))
  }

  "winner" should "return best matching unit (BMU)" in {
    val som = SOM(6, 6)
    som.codeBook.keysIterator.foreach { case (x, y) => som.codeBook(x, y) = Array(0.2, 0.2, 0.2) }
    som.codeBook(3, 3) = Array(0.3, 0.3, 0.3)
    som.winner(new DenseVector(Array(2.0, 2.0, 2.0)), som.codeBook) should equal ((3, 3))
    som.winner(new DenseVector(Array(0.26, 0.26, 0.26)), som.codeBook) should equal ((3, 3))
  }

  "winner" should "return last best matching unit (BMU) index in case of multiple BMUs" in {
    val som = SOM(6, 6)
    som.codeBook.keysIterator.foreach { case (x, y) => som.codeBook(x, y) = Array(0.2, 0.2, 0.2) }
    som.codeBook(3, 3) = Array(0.3, 0.3, 0.3)
    som.winner(new DenseVector(Array(0.25, 0.25, 0.25)), som.codeBook) should equal ((5, 5))
  }

  "classify" should "return the best matching unit along with Euclidean distance" in {
    val som = SOM(6, 6)
    som.codeBook.keysIterator.foreach { case (x, y) => som.codeBook(x, y) = Array(0.2, 0.2, 0.2) }
    som.codeBook(3, 3) = Array(0.3, 0.3, 0.3)
    val (bmu, distance) = som.classify(new DenseVector(Array(0.26, 0.26, 0.26)))
    bmu should === ((3, 3))
    assert(closeTo(distance, 0.06, relDiff = 1e-2))
  }

} 

//设置package包名称以及导入依赖的类
package utils

import breeze.linalg.{DenseMatrix, convert, max, min}
import breeze.plot._

object FloatMatrixUtils {
  def visualize(mat: DenseMatrix[Float]): Figure = {
    val f2 = Figure()
    f2.height = mat.rows
    f2.width = mat.cols
    f2.subplot(0) += image(convert(mat, Double), GradientPaintScale(min(mat), max(mat), PaintScale.BlackToWhite))
    f2.subplot(0).yaxis.setInverted(true)
    f2
  }

  
} 

//设置package包名称以及导入依赖的类
package com.esri

import breeze.linalg.DenseMatrix
import nak.cluster.{DBSCAN, GDBSCAN, Kmeans}

import scala.collection.mutable.ArrayBuffer


object ScanDir {
  def apply(
             iter: Iterable[(Double, Double)],
             epsilon: Double,
             minPoints: Int
           ) = {

    val (xarr, yarr) = iter
      .foldLeft((new ArrayBuffer[Double](), new ArrayBuffer[Double]())) {
        case ((xarr, yarr), (x, y)) => {
          xarr += x
          yarr += y
          (xarr, yarr)
        }
      }

    val matrix = new DenseMatrix[Double](xarr.size, 2, (xarr ++ yarr).toArray)

    val dbScan = new GDBSCAN(
      DBSCAN.getNeighbours(epsilon = epsilon,
        distance = Kmeans.euclideanDistance),
      DBSCAN.isCorePoint(minPoints = minPoints))

    val clusters = dbScan cluster matrix
    clusters.flatMap(cluster => {
      val points = cluster
        .points
        .map(point => {
          val denseVector = point.value
          (denseVector(0), denseVector(1))
        })
      DirDist(points, minPoints)
    })
  }
} 

//设置package包名称以及导入依赖的类
package com.esri

import breeze.linalg.DenseMatrix
import nak.cluster.{DBSCAN, GDBSCAN, Kmeans}

import scala.collection.mutable.ArrayBuffer
import scala.math._


object ScanDist {
  def apply(
             iter: Iterable[(Double, Double)],
             epsilon: Double,
             minPoints: Int
           ): Seq[StdDist] = {

    val (xarr, yarr) = iter
      .foldLeft((new ArrayBuffer[Double](), new ArrayBuffer[Double]())) {
        case ((xarr_, yarr_), (x, y)) => {
          xarr_ += x
          yarr_ += y
          (xarr_, yarr_)
        }
      }

    val matrix = new DenseMatrix[Double](xarr.size, 2, (xarr ++ yarr).toArray)

    val dbScan = new GDBSCAN(
      DBSCAN.getNeighbours(epsilon = epsilon,
        distance = Kmeans.euclideanDistance),
      DBSCAN.isCorePoint(minPoints = minPoints))

    val clusters = dbScan cluster matrix
    clusters.map(cluster => {
      val (ox, oy) = cluster.points.foldLeft((OnlineVar(), OnlineVar())) {
        case ((ox, oy), point) => {
          val denseVector = point.value
          ox += denseVector(0)
          oy += denseVector(1)
          (ox, oy)
        }
      }
      val sd = sqrt(ox.variance + oy.variance)
      StdDist(ox.mu, oy.mu, sd)
    })
  }

} 

//设置package包名称以及导入依赖的类
package hu.sztaki.ilab.ps.test.utils

import breeze.linalg.{DenseMatrix, SparseVector}
import hu.sztaki.ilab.ps.passive.aggressive.algorithm.PassiveAggressiveMulticlassAlgorithm
import org.slf4j.LoggerFactory

class PassiveAggressiveMultiModelEvaluation

object PassiveAggressiveMultiModelEvaluation {

  private val log = LoggerFactory.getLogger(classOf[PassiveAggressiveMultiModelEvaluation])

  def accuracy(model: DenseMatrix[Double], testLines: Traversable[(SparseVector[Double], Option[Int])],
               featureCount: Int, pac: PassiveAggressiveMulticlassAlgorithm): Double = {

    var hit = 0
    var cnt = 0
    testLines.foreach{case(vector, label) => label match {
      case Some(l) =>
      if (pac.predict(vector, model) == l) hit += 1
      cnt += 1
      case _ => throw new IllegalStateException("Labels should not be missing.")
    }}
    val percent = (hit.toDouble / cnt) * 100
    percent
  }

} 

//设置package包名称以及导入依赖的类
package cvx

import breeze.linalg.{DenseMatrix, DenseVector}
import MatrixUtils._


class QuadraticObjectiveFunction(

        override val dim:Int,
        val r:Double,
        val a:DenseVector[Double],
        val P:DenseMatrix[Double]
)
extends ObjectiveFunction(dim) {

    if(a.length!=dim){
        val msg = "Vector a must be of dimension "+dim+" but length(a) "+a.length
        throw new IllegalArgumentException(msg)
    }
    if(!(P.rows==dim & P.cols==dim)) {

        val msg = "Matrix P must be square of dimension "+dim+" but is "+P.rows+"x"+P.cols
        throw new IllegalArgumentException(msg)
    }
    checkSymmetric(P,1e-13)

    def valueAt(x:DenseVector[Double]) = { checkDim(x); r + (a dot x) + (x dot (P*x))/2 }
    def gradientAt(x:DenseVector[Double]) = { checkDim(x); a+P*x }
    def hessianAt(x:DenseVector[Double]) = { checkDim(x); P }

} 

//设置package包名称以及导入依赖的类
package classifiers

import breeze.linalg.{*, DenseMatrix, DenseVector, norm}


  def hinge(xTr: DenseMatrix[Double],
               yTr: DenseVector[Int]): DenseVector[Double] = {
    val doubleYTR = new DenseVector(yTr.toArray.map(_.toDouble))
    val YWX = doubleYTR *:* (xTr(*, ::) dot w)
    val YX = xTr(::, *) *:* doubleYTR
    val delta = (DenseVector.ones[Double](YWX.length) - YWX).map(x => if (x <= 0) 0.0 else 1.0)
    val gradient = (YX(::, *) dot delta) * -1.0
    reg match {
      case Some(reg) => gradient.t + reg.regGradient(w)
      case None => gradient.t
    }
  }

} 

//设置package包名称以及导入依赖的类
package classifiers

import breeze.linalg.{*, DenseMatrix, DenseVector, norm}


  def adagrad(lossFunc: ((DenseMatrix[Double], DenseVector[Int]) => DenseVector[Double]),
                      alpha: Double, maxiter: Int, delta: Double,
                      xTr: DenseMatrix[Double], yTr: DenseVector[Int]): Unit = {
    var z = DenseVector.zeros[Double](dimension)
    for (_ <- 1 until maxiter) {
      val gradient = lossFunc(xTr, yTr)
      z = z + gradient.map {x => x * x}
      val zEps = z :+= 0.0001
      val alphaGradient = gradient :*= alpha
      val newW = w - alphaGradient /:/ zEps.map(x => Math.sqrt(x))
      if (norm(gradient) < delta) return
      w = newW
    }
  }
} 

//设置package包名称以及导入依赖的类
package classifiers

import breeze.linalg.{*, Axis, DenseMatrix, DenseVector, norm, sum}
import breeze.numerics.exp


  def logistic(xTr: DenseMatrix[Double],
               yTr: DenseVector[Int]): DenseVector[Double] = {
    val doubleYTR = new DenseVector(yTr.toArray.map(_.toDouble))
    val YWX = doubleYTR *:* (xTr(*, ::) dot w)
    var eToTheYWX = exp.inPlace(YWX)
    val numerator = xTr(::, *) *:* doubleYTR
    val denominator = eToTheYWX :+= 1.0
    val gradient = (sum(numerator(::, *) /:/ denominator, Axis._0) * -1.0).t
    reg match {
      case Some(reg) => gradient + reg.regGradient(w)
      case None => gradient
    }
  }
} 

//设置package包名称以及导入依赖的类
package classifiers

import breeze.linalg.{DenseMatrix, DenseVector}
import models.Vectorizable



class LinearClassifierWrapper[T <: Vectorizable](val classifier: LinearClassifier) {
  def train(xTr: Seq[T], yTr: Seq[Label]) = {
    val xTrMatrix = seqToMatrix(xTr)
    val yTrVector = seqLabelToVector(yTr)
    classifier.train(xTrMatrix, yTrVector)
  }

  def classify(xTe: Seq[T]): Seq[Label] =
    classifier.classify(seqToMatrix(xTe)).toArray.map(y => BinaryLabel.toLabel(y))

  def test(xTe: Seq[T], yTe: Seq[Label]): Double = {
    classifier.test(seqToMatrix(xTe), seqLabelToVector(yTe))
  }

  def seqToMatrix(seq: Seq[T]): DenseMatrix[Double] = {
    val vectors = seq.map(_.vectorize())
    DenseMatrix.tabulate(vectors.size, vectors.head.length) { case (i, j) => vectors(i).valueAt(j) }
  }

  def seqLabelToVector(seq: Seq[Label]) = DenseVector(seq.map(y => BinaryLabel.toInt(y)).toArray)

  def loadParams(w: DenseVector[Double], b: Double): Unit = {
    try {
      classifier.asInstanceOf[LinearClassifier].loadParams(w, b)
    } catch {
      case _: Exception =>
    }
  }
} 

//设置package包名称以及导入依赖的类
import breeze.linalg.{DenseMatrix, DenseVector}
import classifiers.NaiveBayesClassifier
import org.scalatestplus.play.PlaySpec


class NaiveBayesClassifierSpec extends PlaySpec {
  "Naive bayes classifier"  should {

    "Train w and b correctly and classify correctly" in {
      val classifier = new NaiveBayesClassifier(6)

      val xTr = DenseMatrix(
        (1.0, 1.0, 1.0, 0.0, 0.0, 0.0),
        (1.0, 0.0, 1.0, 0.0, 0.0, 0.0),
        (1.0, 1.0, 0.0, 0.0, 0.0, 0.0),
        (1.0, 1.0, 1.0, 1.0, 0.0, 0.0),
        (1.0, 1.0, 1.0, 0.0, 0.0, 0.0),
        (0.0, 0.0, 0.0, 1.0, 1.0, 1.0),
        (0.0, 0.0, 0.0, 1.0, 0.0, 1.0),
        (0.0, 0.0, 1.0, 0.0, 1.0, 1.0),
        (0.0, 0.0, 0.0, 1.0, 1.0, 1.0),
        (0.0, 1.0, 1.0, 0.0, 1.0, 1.0)
      )

      val yTr = DenseVector(1, 1, 1, 1, 1, -1, -1, -1, -1, -1)

      val xTe = DenseMatrix(
        (1.0, 1.0, 0.0, 0.0, 0.0, 0.0),
        (0.0, 0.0, 0.0, 0.0, 1.0, 1.0),
        (1.0, 1.0, 1.0, 1.0, 0.0, 0.0)
      )

      classifier.train(xTr, yTr)
      val y = classifier.classify(xTe).toArray
      y.length mustBe 3
      y(0) mustBe 1
      y(1) mustBe -1
      y(2) mustBe 1

      val trainingError = classifier.test(xTr, yTr)
      trainingError mustBe 0.0
    }
  }
}