本文整理汇总了Scala中org.apache.spark.graphx.impl.EdgePartitionBuilder类的典型用法代码示例。如果您正苦于以下问题：Scala EdgePartitionBuilder类的具体用法？Scala EdgePartitionBuilder怎么用？Scala EdgePartitionBuilder使用的例子？那么, 这里精选的类代码示例或许可以为您提供帮助。
在下文中一共展示了EdgePartitionBuilder类的1个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Scala代码示例。
示例1: DirectGraphLoader
//设置package包名称以及导入依赖的类
package org.apache.spark.graphx
import org.apache.spark.graphx.impl.EdgePartitionBuilder
import org.apache.spark.graphx.impl.GraphImpl
import org.apache.spark.internal.Logging
import org.apache.spark.rdd.RDD
import org.apache.spark.storage.StorageLevel
import it.unimi.dsi.fastutil.longs.LongOpenHashBigSet
object DirectGraphLoader extends Logging {
def edgeListFile(e: RDD[(Long, LongOpenHashBigSet)]): Graph[Int, Int] =
{
val canonicalOrientation: Boolean = false
val numEdgePartitions: Int = -1
val edgeStorageLevel: StorageLevel = StorageLevel.MEMORY_ONLY
val vertexStorageLevel: StorageLevel = StorageLevel.MEMORY_ONLY
val startTime = System.currentTimeMillis
// Parse the edge data table directly into edge partitions
val lines = e.flatMap { case (vid, adj) => adj.toLongArray().map(dst => vid + "\t" + dst) }
val edges = lines.mapPartitionsWithIndex { (pid, iter) =>
val builder = new EdgePartitionBuilder[Int, Int]
iter.foreach { line =>
if (!line.isEmpty && line(0) != '#') {
val lineArray = line.split("\\s+")
if (lineArray.length < 2) {
throw new IllegalArgumentException("Invalid line: " + line)
}
val srcId = lineArray(0).toLong
val dstId = lineArray(1).toLong
if (canonicalOrientation && srcId > dstId) {
builder.add(dstId, srcId, 1)
} else {
builder.add(srcId, dstId, 1)
}
}
}
Iterator((pid, builder.toEdgePartition))
}.persist(edgeStorageLevel).setName("GraphLoader.edgeListFile - edges")
edges.count()
logInfo("It took %d ms to load the edges".format(System.currentTimeMillis - startTime))
GraphImpl.fromEdgePartitions(edges, defaultVertexAttr = 1, edgeStorageLevel = edgeStorageLevel,
vertexStorageLevel = vertexStorageLevel)
} // end of edgeListFile
}