Scala CSVReader类代码示例

//设置package包名称以及导入依赖的类
package org.hpi.esb.commons.output.writers

import java.io.File

import com.github.tototoshi.csv.CSVReader
import org.hpi.esb.commons.output.CSVOutput
import org.hpi.esb.commons.output.model.Result
import org.hpi.esb.commons.util.Logging

import scala.io.Source

abstract class ResultWriter(inputFilesPrefix: String, resultsPath: String,
                            outputFileName: String) extends Logging {

  def execute(): Unit = {
    val inputSources = getInputSources(resultsPath)
    val table = createResultTable(inputSources)
    writeResults(table, resultsPath, outputFileName)
  }

  def createResultTable(inputSources: List[Source]): List[List[String]] = {
    val intermediateResultMaps = getIntermediateResultMaps(inputSources)
    if (intermediateResultMaps.nonEmpty) {
      val finalResult = getFinalResult(intermediateResultMaps)
      finalResult.toTable()
    } else {
      logger.info("No benchmark series result files were found for merger.")
      sys.exit(1)
    }
  }

  def writeResults(mergedResults: List[List[String]], resultPath: String, fileName: String): Unit = {
    CSVOutput.write(mergedResults, resultPath, fileName)
  }

  def getIntermediateResultMaps(sources: List[Source]): List[Map[String, String]] = {
    val readers = sources.map(CSVReader.open)
    readers.flatMap(r => r.allWithHeaders())
  }

  def getInputSources(resultsPath: String): List[Source] = {
    val files = getListOfFiles(resultsPath)
    val filteredFiles = filterFilesByPrefix(files, inputFilesPrefix)
    filteredFiles.map(Source.fromFile)
  }

  def filterFilesByPrefix(files: List[File], prefix: String): List[File] = {
    files.filter(_.getName.startsWith(prefix))
  }

  def getListOfFiles(dir: String): List[File] = {
    val d = new File(dir)
    if (d.exists && d.isDirectory) {
      d.listFiles.filter(_.isFile).toList
    } else {
      List[File]()
    }
  }

  def getFinalResult(runResultMaps: List[Map[String, String]]): Result
} 

//设置package包名称以及导入依赖的类
package task.airport
package data

import java.net.URL

import akka.NotUsed
import akka.stream.scaladsl.Source
import com.github.tototoshi.csv.CSVReader
import task.airport.model.{Airport, Country, Runway}

import scala.io.{Source => IOSource}
import scala.util.{Failure, Success, Try}

object Data {

  private val log = Logging logger getClass

  lazy val countries: Source[Country, NotUsed] =
    source(resources.countriesCSV)(Parser.country)

  lazy val airports: Source[Airport, NotUsed] =
    source(resources.airportsCSV)(Parser.airport)

  lazy val runways: Source[Runway, NotUsed] =
    source(resources.runwaysCSV)(Parser.runway)

  private def source[T](url: URL)(parse: Seq[String] => T): Source[T, NotUsed] =
    Source
      .fromIterator { () =>
        val source = IOSource fromURL url
        val reader = CSVReader open source
        val iterator = reader.iterator
        iterator
      }
      .drop(1)
      .map { row =>
        Try {
          parse(row)
        } match {
          case Success(value) =>
            value
          case Failure(exception) =>
            // It might be dangerous to log the CSV row.
            log.warn(s"Failed to parse: $row")
            throw exception
        }
      }

} 

//设置package包名称以及导入依赖的类
package sug.batchservice

import java.io.File

import com.github.tototoshi.csv.CSVReader
import com.typesafe.scalalogging.StrictLogging
import org.apache.commons.io.monitor.{FileAlterationListenerAdaptor, FileAlterationMonitor, FileAlterationObserver}

object BatchService extends App with StrictLogging {

  def processor(person: Person): Person = person match {
    case Person(fn, ln) => Person(fn.toUpperCase,ln.toUpperCase)
  }

  case class Person(firstName: String, lastName: String) {
    override def toString: String = s"firstName: $firstName lastName: $lastName"
  }

  def monitor(directory: String, interval: Long = 1000) = {
    val m = new FileAlterationMonitor(interval)
    val o = new FileAlterationObserver(directory)
    m.addObserver(o)
    m.start()

    val l = new FileAlterationListenerAdaptor {
      override def onFileChange(file: File): Unit = {
        logger.info(s"File changed ${file.getName}.")
        //TODO
      }
    }
    o.addListener(l)
  }

  def reader(absolutePath: String): List[List[String]] = {
    val reader = CSVReader.open(new File(absolutePath))
    val rows = reader.all()
    logger.info(s"Read ${rows.size} row.")
    rows.map(_.map(_.trim))
  }

  def converter(raw: List[String]): Either[List[String], Person] = raw match {
    case firstName :: lastName :: Nil if lastName.nonEmpty && firstName.nonEmpty =>
      Right(Person(firstName, lastName))
    case row => Left(row)
  }

  def errorWriter(err: List[String]): Unit =
    logger.info(s"Err: $err")


  def okWriter(person: Person): Unit =
    logger.info(s"Writer: $person")
} 

//设置package包名称以及导入依赖的类
package util

import java.io.{BufferedWriter, File, FileOutputStream, FileWriter}
import java.nio.file.{Files, Paths}

import com.github.tototoshi.csv.{CSVReader, CSVWriter, DefaultCSVFormat}
import com.google.inject.Singleton

import scala.io.Source

@Singleton
class LocalDataRepo extends BaseDataRepo {

  class CSVFormat(_delimiter: Char) extends DefaultCSVFormat {
    override val delimiter: Char = _delimiter
  }

  override def read(filePath: String): Array[Byte] = {
    Files.readAllBytes(Paths.get(filePath))
  }

  override def readCsv(filePath: String, delimiter: Char): Seq[Seq[String]] = {
    CSVReader.open(new File(filePath))(new CSVFormat(delimiter)).all()
  }

  override def readCsvAsMap(filePath: String, delimiter: Char): Seq[Map[String, String]] = {
    CSVReader.open(new File(filePath))(new CSVFormat(delimiter)).allWithHeaders()
  }

  override def readLines(filePath: String): Seq[String] = {
    Source.fromFile(filePath).getLines().toList
  }

  override def write(data: Array[Byte], filePath: String): Unit = {
    new FileOutputStream(new File(filePath)).write(data)
  }

  override def writeCsv(data: Seq[Seq[Any]], filePath: String, delimiter: Char): Unit = {
    CSVWriter.open(new File(filePath))(new CSVFormat(delimiter)).writeAll(data)
  }

  override def writeLines(data: Seq[String], filePath: String): Unit = {
    new BufferedWriter(new FileWriter(filePath)).write(data.mkString(scala.util.Properties.lineSeparator))
  }

} 

//设置package包名称以及导入依赖的类
package com.miriamlaurel.aggro.csv.okcoin

import java.io.{File, FilenameFilter}
import java.time.{Instant, ZoneId}
import java.time.format.DateTimeFormatter
import java.time.temporal.TemporalAccessor
import java.util.UUID

import com.github.tototoshi.csv.CSVReader
import com.miriamlaurel.aggro.Inventory
import com.miriamlaurel.aggro.model.Fill
import com.miriamlaurel.fxcore._
import com.miriamlaurel.fxcore.asset.Currency
import com.miriamlaurel.fxcore.party.Party
import com.miriamlaurel.fxcore.portfolio.Position

object CsvLoader {

  val oft = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss").withZone(ZoneId.of("Asia/Shanghai"))

  def parseCsv(tokens: Seq[String]): Fill = {
    val fillId = tokens.head.toLong
    val id = tokens(1).toLong
    val timeString: String = tokens(2)
    val btcBalance = SafeDouble(tokens(5).split("/")(0).replaceAll(",", "").toDouble)
    val cnyBalance = SafeDouble(tokens(7).replaceAll(",", "").toDouble)
    val parsedTime: TemporalAccessor = oft.parse(timeString)
    val ts = Instant.from(parsedTime)
    try {
      val btcStr = tokens(4).replaceAll(",", "")
      val btcS = if (btcStr.startsWith("+")) btcStr.substring(1) else btcStr
      val cnyStr = tokens(6).replaceAll(",", "")
      val cnyS = if (cnyStr.startsWith("+")) cnyStr.substring(1) else cnyStr
      val btc: Monetary = Monetary(SafeDouble(btcS.toDouble), Bitcoin)
      val cny: Monetary = Monetary(SafeDouble(cnyS.toDouble), Currency("CNY"))
      val position = Position(btc, cny, None, ts.toEpochMilli, UUID.randomUUID())
      val inventory: Inventory = Map(Bitcoin -> btcBalance, Currency("CNY") -> cnyBalance)
      Fill(Party("OKCN"), position, id.toString, Some(fillId.toString), Some(inventory))
    } catch {
      case x: Throwable =>
        x.printStackTrace()
        throw x
    }

  }

  def loadFromCsv(dir: File): Array[Fill] = {
    val files = dir.listFiles(new FilenameFilter {
      override def accept(dir: File, name: String): Boolean = name.endsWith(".csv")
    })
    val data = for (file <- files) yield {
      val reader = CSVReader.open(file)
      val fills = reader.toStream.drop(1).filter(line => {
        !line.head.startsWith("1024548491")
      }).map(parseCsv).toArray
      reader.close()
      fills
    }
    data.flatten
  }
} 

//设置package包名称以及导入依赖的类
package funchisqwrapper.parsing

import java.io.File

import com.github.tototoshi.csv.{CSVReader, TSVFormat}
import funchisqwrapper.TimeSeriesTypes.{ContinuousProfile, ContinuousTimeSeries}

object TimeSeriesParser {
  private val ID_LABEL = "id"

  def parse(f: File): ContinuousTimeSeries = {
    val format = new TSVFormat {}
    val reader = CSVReader.open(f)(format)
    val (headers, rows) = reader.allWithOrderedHeaders()

    assert(headers.size > 1)
    assert(headers.head == ID_LABEL)

    val labels = headers.tail

    val profiles = rows map { row =>
      val id = row(ID_LABEL)
      val values = labels map { l => row(l).toDouble }
      ContinuousProfile(id, values)
    }

    ContinuousTimeSeries(profiles)
  }
} 

//设置package包名称以及导入依赖的类
package services

import java.io.File
import javax.inject._

import com.github.tototoshi.csv.{CSVReader, DefaultCSVFormat}
import play.api.{Configuration, Logger}


@Singleton
class FileSystem @Inject() (configuration: Configuration) {

  val repodir = configuration.underlying.getString("importnow.dir")
  val uploadsDir = new File(repodir + "/uploads")
  uploadsDir.mkdirs()
  Logger.info(s"Saving upload files to $uploadsDir")

  def getUploads: List[String] = {
    uploadsDir.list().toList.sorted
  }

  def uploadFile(filename: String, contentType: Option[String])(writeTo: File => Unit): Unit = {
    val file = new File(uploadsDir, filename)
    writeTo(file)
  }

  def fileHeaders(filename: String): Option[List[String]] = {
    withCsvReader(filename) {
      reader => reader.readNext()
    }
  }

  def fileStream[T](filename: String)(streamFunction: Stream[Map[String, String]] => T): T = {
    withCsvReader(filename) {
      reader => streamFunction(reader.toStreamWithHeaders)
    }
  }

  private def withCsvReader[T](filename: String)(function: CSVReader => T): T = {
    val file = new File(uploadsDir, filename)

    implicit object MyFormat extends DefaultCSVFormat {
      override val delimiter = ';'
    }

    val reader = CSVReader.open(file)
    val result = try {
      function(reader)
    } finally {
      reader.close
    }
    result
  }

} 

//设置package包名称以及导入依赖的类
package com.carl.ts.examples

import java.io.File
import java.time.LocalDateTime

import com.github.tototoshi.csv.CSVReader
import com.carl.ts.Series
import com.carl.ts.Stats.SeriesStats



object AnomalyApp {

  def main(args: Array[String]): Unit = {
    val series = loadCSV("testdata/anomalies.csv")
    printStats("Raw data", series)
    printStats("Differences", series.differentiate)
  }

  def loadCSV(fileName: String): Series[Double] = {
    import java.time.format.DateTimeFormatter
    val format = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")
    val reader = CSVReader.open(new File(fileName))
    val data = reader.allWithHeaders()
    val readings: List[(LocalDateTime, Double)] = data.map(row => {
      val index = row.get("time").map(x => LocalDateTime.parse(x,format)).getOrElse(LocalDateTime.now())
      val value = row.get("velocity").map(_.toDouble).getOrElse(0.0)
      (index, value)
    })
    new Series(readings)
  }

  def printStats(title: String, series: Series[Double]) = {
    println(title)
    println("  * Length: " + series.length)
    println("  * Mean: " + series.mean)
    println("  * Std dev: " + series.stddev)

  }
} 

//设置package包名称以及导入依赖的类
package Readers

import java.io.File
import java.time.Month

import Core.{Availability, Person}
import com.github.tototoshi.csv.CSVReader

case class VolunteerCSVReader(filename: String) {
  private val reader = CSVReader.open(new File(filename))
  private val raw: List[Map[String, String]] = reader.allWithHeaders()

  private def availability(row: Map[String,String], tentativeMeansAvailable: Boolean) : Availability = {
    val sun = "Available Sun 6/12"    -> Availability.AllDay(2016, Month.JUNE, 12)
    val mon = "Available Mon 6/13"    -> Availability.AllDay(2016, Month.JUNE, 13)
    val tue = "Available Tue 6/14"    -> Availability.AllDay(2016, Month.JUNE, 14)
    val wed = "Available Wed 6/15"    -> Availability.AllDay(2016, Month.JUNE, 15)
    val thu = "Available Thurs 6/16"  -> Availability.AllDay(2016, Month.JUNE, 16)
    val fri = "Available Fri 6/17"    -> Availability.AllDay(2016, Month.JUNE, 17)

    val days = List(sun, mon, tue, wed, thu, fri)

    // compute availability
    days.foldLeft(Availability.NotAvailable){ case (acc,(key,avail)) =>
        if (row(key) == "Y"
            || (if (tentativeMeansAvailable) { row(key) == "T" } else { false })) {
          acc + avail
        } else {
          acc
        }
    }
  }

  def people(tentativeMeansAvailable: Boolean) : (Set[Person],Set[Person]) = {
    val allPeople = raw.flatMap { row =>
      if (row("Can Serve (confirmed)") == "Y") {
        val avail = availability(row, tentativeMeansAvailable)

        Some(
          Person(
            row("First name"),
            row("Last name"),
            avail
          )
        )
      } else {
        None
      }
    }.toSet

    allPeople.partition { person => person.availability != Availability.NotAvailable }
  }
}