本文整理汇总了Scala中org.apache.spark.sql.DataFrameReader类的典型用法代码示例。如果您正苦于以下问题：Scala DataFrameReader类的具体用法？Scala DataFrameReader怎么用？Scala DataFrameReader使用的例子？那么恭喜您, 这里精选的类代码示例或许可以为您提供帮助。
在下文中一共展示了DataFrameReader类的3个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Scala代码示例。
示例1: DataFrameReaderConfigurator
//设置package包名称以及导入依赖的类
package ml.sparkling.graph.loaders.csv
import org.apache.spark.sql.DataFrameReader
object DataFrameReaderConfigurator {
implicit class addAbilityToConfigureDataFrameReader(reader:DataFrameReader){
def applyConfiguration(cSVLoaderConfig: CsvLoaderConfig):DataFrameReader={
reader.option("header",cSVLoaderConfig.header.toString)
reader.option("delimiter",cSVLoaderConfig.delimiter)
reader.option("quote",cSVLoaderConfig.quote)
cSVLoaderConfig.schema match{
case None => reader.option("inferSchema",cSVLoaderConfig.inferSchema.toString)
case Some(schema) => reader.schema(schema)
}
reader
}
}
} 示例2:
//设置package包名称以及导入依赖的类
package test.yumi.pipeline
import com.typesafe.config.Config
import org.apache.spark.SparkContext
import org.apache.spark.sql.{DataFrameReader, SparkSession}
import org.apache.spark.sql.types.StructType
import org.mockito.invocation.InvocationOnMock
import org.mockito.stubbing.Answer
import yumi.Job
import yumi.metastore.Metastore
import yumi.pipeline._
trait MockSessionSpec extends BaseSpec {
trait MockSessionScope extends BaseMockScope {
implicit val yumiContext = mock[YumiContext]
val pipelineFactory = mock[PipelineFactory]
val activityLoader = mock[ActivityLoader]
val contextFactory = mock[YumiContextFactory]
val sparkSession = mock[SparkSession]
val sparkContext = mock[SparkContext]
val dataFrameReader = mock[DataFrameReader]
val dataFrameWriter = mock[DataFrameWriter]
val emptyParameters = new Parameters
when(sparkSession.sparkContext).thenReturn(sparkContext)
when(yumiContext.sparkSession).thenReturn(sparkSession)
when(yumiContext.sparkContext).thenReturn(sparkContext)
when(yumiContext.dataFrameWriter).thenReturn(dataFrameWriter)
when(sparkSession.read).thenReturn(dataFrameReader)
when(dataFrameReader.format(any[String])).thenReturn(dataFrameReader)
when(dataFrameReader.schema(any[StructType])).thenReturn(dataFrameReader)
when(dataFrameReader.option(any[String], any[String])).thenReturn(dataFrameReader)
when(dataFrameReader.options(any[Map[String, String]])).thenReturn(dataFrameReader)
}
} 示例3: DynamoDBDataFrameReader
//设置package包名称以及导入依赖的类
package com.github.traviscrawford.spark
import org.apache.spark.sql.DataFrame
import org.apache.spark.sql.DataFrameReader
package object dynamodb {
implicit class DynamoDBDataFrameReader(reader: DataFrameReader) {
def dynamodb(region: String, table: String): DataFrame =
reader
.format("com.github.traviscrawford.spark.dynamodb")
.option("region", region)
.option("table", table)
.load
}
}