本文整理汇总了Scala中org.apache.spark.rdd.UnionRDD类的典型用法代码示例。如果您正苦于以下问题：Scala UnionRDD类的具体用法？Scala UnionRDD怎么用？Scala UnionRDD使用的例子？那么恭喜您, 这里精选的类代码示例或许可以为您提供帮助。
在下文中一共展示了UnionRDD类的1个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Scala代码示例。
示例1: TestableQueueInputDStream
//设置package包名称以及导入依赖的类
package org.apache.spark.streaming
import java.io.{ObjectInputStream, ObjectOutputStream}
import org.apache.spark.rdd.{RDD, UnionRDD}
import org.apache.spark.streaming.dstream.InputDStream
import scala.collection.mutable.{ArrayBuffer, Queue}
import scala.reflect.ClassTag
class TestableQueueInputDStream[T: ClassTag](
ssc: StreamingContext,
val queue: Queue[RDD[T]],
oneAtATime: Boolean,
defaultRDD: RDD[T]
) extends InputDStream[T](ssc) {
override def start() { }
override def stop() { }
private def readObject(in: ObjectInputStream): Unit = {
logWarning("queueStream doesn't support checkpointing")
}
private def writeObject(oos: ObjectOutputStream): Unit = {
logWarning("queueStream doesn't support checkpointing")
}
override def compute(validTime: Time): Option[RDD[T]] = {
val buffer = new ArrayBuffer[RDD[T]]()
queue.synchronized {
if (oneAtATime && queue.nonEmpty) {
buffer += queue.dequeue()
} else {
buffer ++= queue
queue.clear()
}
}
if (buffer.nonEmpty) {
if (oneAtATime) {
Some(buffer.head)
} else {
Some(new UnionRDD(context.sc, buffer.toSeq))
}
} else if (defaultRDD != null) {
Some(defaultRDD)
} else {
Some(ssc.sparkContext.emptyRDD)
}
}
}