本文整理汇总了Java中org.apache.spark.storage.BlockManagerId类的典型用法代码示例。如果您正苦于以下问题：Java BlockManagerId类的具体用法？Java BlockManagerId怎么用？Java BlockManagerId使用的例子？那么, 这里精选的类代码示例或许可以为您提供帮助。
BlockManagerId类属于org.apache.spark.storage包，在下文中一共展示了BlockManagerId类的4个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Java代码示例。
示例1: SparkIoRegistry
import org.apache.spark.storage.BlockManagerId; //导入依赖的package包/类
private SparkIoRegistry() {
try {
super.register(GryoIo.class, Tuple2.class, new Tuple2Serializer());
super.register(GryoIo.class, Tuple2[].class, null);
super.register(GryoIo.class, Tuple3.class, new Tuple3Serializer());
super.register(GryoIo.class, Tuple3[].class, null);
super.register(GryoIo.class, CompactBuffer.class, new CompactBufferSerializer());
super.register(GryoIo.class, CompactBuffer[].class, null);
super.register(GryoIo.class, CompressedMapStatus.class, null);
super.register(GryoIo.class, BlockManagerId.class, null);
super.register(GryoIo.class, HighlyCompressedMapStatus.class, new ExternalizableSerializer()); // externalizable implemented so its okay
super.register(GryoIo.class, TorrentBroadcast.class, null);
super.register(GryoIo.class, PythonBroadcast.class, null);
super.register(GryoIo.class, BoxedUnit.class, null);
super.register(GryoIo.class, Class.forName("scala.reflect.ClassTag$$anon$1"), new JavaSerializer());
super.register(GryoIo.class, Class.forName("scala.reflect.ManifestFactory$$anon$1"), new JavaSerializer());
super.register(GryoIo.class, Class.forName("org.apache.spark.internal.io.FileCommitProtocol$TaskCommitMessage"), new JavaSerializer());
super.register(GryoIo.class, Class.forName("org.apache.spark.internal.io.FileCommitProtocol$EmptyTaskCommitMessage$"), new JavaSerializer());
super.register(GryoIo.class, Class.forName("scala.collection.immutable.Map$EmptyMap$"), new JavaSerializer());
super.register(GryoIo.class, Class.forName("scala.collection.immutable.Map"), new JavaSerializer());
super.register(GryoIo.class, Class.forName("scala.None$"), new JavaSerializer());
super.register(GryoIo.class, Class.forName("scala.Some$"), new JavaSerializer());
super.register(GryoIo.class, Class.forName("scala.Some"), new JavaSerializer());
super.register(GryoIo.class, WrappedArray.ofRef.class, new WrappedArraySerializer());
super.register(GryoIo.class, MessagePayload.class, null);
super.register(GryoIo.class, ViewIncomingPayload.class, null);
super.register(GryoIo.class, ViewOutgoingPayload.class, null);
super.register(GryoIo.class, ViewPayload.class, null);
super.register(GryoIo.class, SerializableConfiguration.class, new JavaSerializer());
super.register(GryoIo.class, VertexWritable.class, new VertexWritableSerializer());
super.register(GryoIo.class, ObjectWritable.class, new ObjectWritableSerializer());
} catch (final ClassNotFoundException e) {
throw new IllegalStateException(e);
}
}
示例2: onBlockManagerAdded
import org.apache.spark.storage.BlockManagerId; //导入依赖的package包/类
@Override
public void onBlockManagerAdded(SparkListenerBlockManagerAdded blockManagerAdded) {
final BlockManagerId bmId = blockManagerAdded.blockManagerId();
writeEvent("blockManagerAdded", blockManagerAdded.time(), bmId.host(), bmId.executorId(), blockManagerAdded);
}
示例3: onBlockManagerRemoved
import org.apache.spark.storage.BlockManagerId; //导入依赖的package包/类
@Override
public void onBlockManagerRemoved(SparkListenerBlockManagerRemoved blockManagerRemoved) {
final BlockManagerId bmId = blockManagerRemoved.blockManagerId();
writeEvent("blockManagerRemoved", blockManagerRemoved.time(), bmId.host(), bmId.executorId(),
blockManagerRemoved);
}
示例4: GryoSerializer
import org.apache.spark.storage.BlockManagerId; //导入依赖的package包/类
public GryoSerializer(final SparkConf sparkConfiguration) {
final long bufferSizeKb = sparkConfiguration.getSizeAsKb("spark.kryoserializer.buffer", "64k");
final long maxBufferSizeMb = sparkConfiguration.getSizeAsMb("spark.kryoserializer.buffer.max", "64m");
final boolean referenceTracking = sparkConfiguration.getBoolean("spark.kryo.referenceTracking", true);
final boolean registrationRequired = sparkConfiguration.getBoolean("spark.kryo.registrationRequired", false);
if (bufferSizeKb >= ByteUnit.GiB.toKiB(2L)) {
throw new IllegalArgumentException("spark.kryoserializer.buffer must be less than 2048 mb, got: " + bufferSizeKb + " mb.");
} else {
this.bufferSize = (int) ByteUnit.KiB.toBytes(bufferSizeKb);
if (maxBufferSizeMb >= ByteUnit.GiB.toMiB(2L)) {
throw new IllegalArgumentException("spark.kryoserializer.buffer.max must be less than 2048 mb, got: " + maxBufferSizeMb + " mb.");
} else {
this.maxBufferSize = (int) ByteUnit.MiB.toBytes(maxBufferSizeMb);
//this.userRegistrator = sparkConfiguration.getOption("spark.kryo.registrator");
}
}
this.gryoPool = GryoPool.build().
poolSize(sparkConfiguration.getInt(GryoPool.CONFIG_IO_GRYO_POOL_SIZE, GryoPool.CONFIG_IO_GRYO_POOL_SIZE_DEFAULT)).
ioRegistries(makeApacheConfiguration(sparkConfiguration).getList(GryoPool.CONFIG_IO_REGISTRY, Collections.emptyList())).
initializeMapper(builder -> {
try {
builder.addCustom(Tuple2.class, new Tuple2Serializer())
.addCustom(Tuple2[].class)
.addCustom(Tuple3.class, new Tuple3Serializer())
.addCustom(Tuple3[].class)
.addCustom(CompactBuffer.class, new CompactBufferSerializer())
.addCustom(CompactBuffer[].class)
.addCustom(CompressedMapStatus.class)
.addCustom(BlockManagerId.class)
.addCustom(HighlyCompressedMapStatus.class, new ExternalizableSerializer()) // externalizable implemented so its okay
.addCustom(HttpBroadcast.class)
.addCustom(PythonBroadcast.class)
.addCustom(BoxedUnit.class)
.addCustom(Class.forName("scala.reflect.ClassTag$$anon$1"), new JavaSerializer())
.addCustom(Class.forName("scala.reflect.ManifestFactory$$anon$1"), new JavaSerializer())
.addCustom(WrappedArray.ofRef.class, new WrappedArraySerializer())
.addCustom(MessagePayload.class)
.addCustom(ViewIncomingPayload.class)
.addCustom(ViewOutgoingPayload.class)
.addCustom(ViewPayload.class)
.addCustom(SerializableConfiguration.class, new JavaSerializer())
.addCustom(VertexWritable.class, new VertexWritableSerializer())
.addCustom(ObjectWritable.class, new ObjectWritableSerializer())
.referenceTracking(referenceTracking)
.registrationRequired(registrationRequired);
// add these as we find ClassNotFoundExceptions
} catch (final ClassNotFoundException e) {
throw new IllegalStateException(e);
}
}).create();
}