本文整理汇总了Java中org.apache.flink.api.common.ExecutionConfig.registerKryoType方法的典型用法代码示例。如果您正苦于以下问题:Java ExecutionConfig.registerKryoType方法的具体用法?Java ExecutionConfig.registerKryoType怎么用?Java ExecutionConfig.registerKryoType使用的例子?那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类org.apache.flink.api.common.ExecutionConfig
的用法示例。
在下文中一共展示了ExecutionConfig.registerKryoType方法的13个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Java代码示例。
示例1: addStormConfigToTopology
import org.apache.flink.api.common.ExecutionConfig; //导入方法依赖的package包/类
@SuppressWarnings({ "unchecked", "rawtypes" })
static void addStormConfigToTopology(FlinkTopology topology, Map conf) throws ClassNotFoundException {
if (conf != null) {
ExecutionConfig flinkConfig = topology.getExecutionEnvironment().getConfig();
flinkConfig.setGlobalJobParameters(new StormConfig(conf));
// add all registered types to ExecutionConfig
List<?> registeredClasses = (List<?>) conf.get(Config.TOPOLOGY_KRYO_REGISTER);
if (registeredClasses != null) {
for (Object klass : registeredClasses) {
if (klass instanceof String) {
flinkConfig.registerKryoType(Class.forName((String) klass));
} else {
for (Entry<String, String> register : ((Map<String, String>) klass).entrySet()) {
flinkConfig.registerTypeWithKryoSerializer(Class.forName(register.getKey()),
(Class<? extends Serializer<?>>) Class.forName(register.getValue()));
}
}
}
}
}
}
示例2: testValueStateDescriptorLazySerializer
import org.apache.flink.api.common.ExecutionConfig; //导入方法依赖的package包/类
@Test
public void testValueStateDescriptorLazySerializer() throws Exception {
// some default value that goes to the generic serializer
Path defaultValue = new Path(new File(ConfigConstants.DEFAULT_TASK_MANAGER_TMP_PATH).toURI());
// some different registered value
ExecutionConfig cfg = new ExecutionConfig();
cfg.registerKryoType(TaskInfo.class);
ValueStateDescriptor<Path> descr =
new ValueStateDescriptor<Path>("testName", Path.class, defaultValue);
try {
descr.getSerializer();
fail("should cause an exception");
} catch (IllegalStateException ignored) {}
descr.initializeSerializerUnlessSet(cfg);
assertNotNull(descr.getSerializer());
assertTrue(descr.getSerializer() instanceof KryoSerializer);
assertTrue(((KryoSerializer<?>) descr.getSerializer()).getKryo().getRegistration(TaskInfo.class).getId() > 0);
}
示例3: testValueStateDescriptorLazySerializer
import org.apache.flink.api.common.ExecutionConfig; //导入方法依赖的package包/类
@Test
public void testValueStateDescriptorLazySerializer() throws Exception {
@SuppressWarnings("unchecked")
ReduceFunction<Path> reducer = mock(ReduceFunction.class);
// some different registered value
ExecutionConfig cfg = new ExecutionConfig();
cfg.registerKryoType(TaskInfo.class);
ReducingStateDescriptor<Path> descr =
new ReducingStateDescriptor<Path>("testName", reducer, Path.class);
try {
descr.getSerializer();
fail("should cause an exception");
} catch (IllegalStateException ignored) {}
descr.initializeSerializerUnlessSet(cfg);
assertNotNull(descr.getSerializer());
assertTrue(descr.getSerializer() instanceof KryoSerializer);
assertTrue(((KryoSerializer<?>) descr.getSerializer()).getKryo().getRegistration(TaskInfo.class).getId() > 0);
}
示例4: testValueStateDescriptorLazySerializer
import org.apache.flink.api.common.ExecutionConfig; //导入方法依赖的package包/类
@Test
public void testValueStateDescriptorLazySerializer() throws Exception {
// some different registered value
ExecutionConfig cfg = new ExecutionConfig();
cfg.registerKryoType(TaskInfo.class);
ListStateDescriptor<Path> descr =
new ListStateDescriptor<Path>("testName", Path.class);
try {
descr.getSerializer();
fail("should cause an exception");
} catch (IllegalStateException ignored) {}
descr.initializeSerializerUnlessSet(cfg);
assertNotNull(descr.getSerializer());
assertTrue(descr.getSerializer() instanceof ListSerializer);
assertNotNull(descr.getElementSerializer());
assertTrue(descr.getElementSerializer() instanceof KryoSerializer);
assertTrue(((KryoSerializer<?>) descr.getElementSerializer()).getKryo().getRegistration(TaskInfo.class).getId() > 0);
}
示例5: testValueStateInstantiation
import org.apache.flink.api.common.ExecutionConfig; //导入方法依赖的package包/类
@Test
public void testValueStateInstantiation() throws Exception {
final ExecutionConfig config = new ExecutionConfig();
config.registerKryoType(Path.class);
final AtomicReference<Object> descriptorCapture = new AtomicReference<>();
StreamingRuntimeContext context = new StreamingRuntimeContext(
createDescriptorCapturingMockOp(descriptorCapture, config),
createMockEnvironment(),
Collections.<String, Accumulator<?, ?>>emptyMap());
ValueStateDescriptor<TaskInfo> descr = new ValueStateDescriptor<>("name", TaskInfo.class);
context.getState(descr);
StateDescriptor<?, ?> descrIntercepted = (StateDescriptor<?, ?>) descriptorCapture.get();
TypeSerializer<?> serializer = descrIntercepted.getSerializer();
// check that the Path class is really registered, i.e., the execution config was applied
assertTrue(serializer instanceof KryoSerializer);
assertTrue(((KryoSerializer<?>) serializer).getKryo().getRegistration(Path.class).getId() > 0);
}
示例6: testListStateInstantiation
import org.apache.flink.api.common.ExecutionConfig; //导入方法依赖的package包/类
@Test
public void testListStateInstantiation() throws Exception {
final ExecutionConfig config = new ExecutionConfig();
config.registerKryoType(Path.class);
final AtomicReference<Object> descriptorCapture = new AtomicReference<>();
StreamingRuntimeContext context = new StreamingRuntimeContext(
createDescriptorCapturingMockOp(descriptorCapture, config),
createMockEnvironment(),
Collections.<String, Accumulator<?, ?>>emptyMap());
ListStateDescriptor<TaskInfo> descr = new ListStateDescriptor<>("name", TaskInfo.class);
context.getListState(descr);
ListStateDescriptor<?> descrIntercepted = (ListStateDescriptor<?>) descriptorCapture.get();
TypeSerializer<?> serializer = descrIntercepted.getSerializer();
// check that the Path class is really registered, i.e., the execution config was applied
assertTrue(serializer instanceof ListSerializer);
TypeSerializer<?> elementSerializer = descrIntercepted.getElementSerializer();
assertTrue(elementSerializer instanceof KryoSerializer);
assertTrue(((KryoSerializer<?>) elementSerializer).getKryo().getRegistration(Path.class).getId() > 0);
}
示例7: testMapStateInstantiation
import org.apache.flink.api.common.ExecutionConfig; //导入方法依赖的package包/类
@Test
public void testMapStateInstantiation() throws Exception {
final ExecutionConfig config = new ExecutionConfig();
config.registerKryoType(Path.class);
final AtomicReference<Object> descriptorCapture = new AtomicReference<>();
StreamingRuntimeContext context = new StreamingRuntimeContext(
createDescriptorCapturingMockOp(descriptorCapture, config),
createMockEnvironment(),
Collections.<String, Accumulator<?, ?>>emptyMap());
MapStateDescriptor<String, TaskInfo> descr =
new MapStateDescriptor<>("name", String.class, TaskInfo.class);
context.getMapState(descr);
MapStateDescriptor<?, ?> descrIntercepted = (MapStateDescriptor<?, ?>) descriptorCapture.get();
TypeSerializer<?> valueSerializer = descrIntercepted.getValueSerializer();
// check that the Path class is really registered, i.e., the execution config was applied
assertTrue(valueSerializer instanceof KryoSerializer);
assertTrue(((KryoSerializer<?>) valueSerializer).getKryo().getRegistration(Path.class).getId() > 0);
}
示例8: recursivelyRegisterType
import org.apache.flink.api.common.ExecutionConfig; //导入方法依赖的package包/类
public static void recursivelyRegisterType(Class<?> type, ExecutionConfig config, Set<Class<?>> alreadySeen) {
// don't register or remember primitives
if (type == null || type.isPrimitive() || type == Object.class) {
return;
}
// prevent infinite recursion for recursive types
if (!alreadySeen.add(type)) {
return;
}
if (type.isArray()) {
recursivelyRegisterType(type.getComponentType(), config, alreadySeen);
}
else {
config.registerKryoType(type);
// add serializers for Avro type if necessary
AvroUtils.getAvroUtils().addAvroSerializersIfRequired(config, type);
Field[] fields = type.getDeclaredFields();
for (Field field : fields) {
if (Modifier.isStatic(field.getModifiers()) || Modifier.isTransient(field.getModifiers())) {
continue;
}
Type fieldType = field.getGenericType();
recursivelyRegisterGenericType(fieldType, config, alreadySeen);
}
}
}
示例9: testMapStateDescriptorLazySerializer
import org.apache.flink.api.common.ExecutionConfig; //导入方法依赖的package包/类
@Test
public void testMapStateDescriptorLazySerializer() throws Exception {
// some different registered value
ExecutionConfig cfg = new ExecutionConfig();
cfg.registerKryoType(TaskInfo.class);
MapStateDescriptor<Path, String> descr =
new MapStateDescriptor<>("testName", Path.class, String.class);
try {
descr.getSerializer();
fail("should cause an exception");
} catch (IllegalStateException ignored) {}
descr.initializeSerializerUnlessSet(cfg);
assertNotNull(descr.getSerializer());
assertTrue(descr.getSerializer() instanceof MapSerializer);
assertNotNull(descr.getKeySerializer());
assertTrue(descr.getKeySerializer() instanceof KryoSerializer);
assertTrue(((KryoSerializer<?>) descr.getKeySerializer()).getKryo().getRegistration(TaskInfo.class).getId() > 0);
assertNotNull(descr.getValueSerializer());
assertTrue(descr.getValueSerializer() instanceof StringSerializer);
}
示例10: testReducingStateInstantiation
import org.apache.flink.api.common.ExecutionConfig; //导入方法依赖的package包/类
@Test
public void testReducingStateInstantiation() throws Exception {
final ExecutionConfig config = new ExecutionConfig();
config.registerKryoType(Path.class);
final AtomicReference<Object> descriptorCapture = new AtomicReference<>();
StreamingRuntimeContext context = new StreamingRuntimeContext(
createDescriptorCapturingMockOp(descriptorCapture, config),
createMockEnvironment(),
Collections.<String, Accumulator<?, ?>>emptyMap());
@SuppressWarnings("unchecked")
ReduceFunction<TaskInfo> reducer = (ReduceFunction<TaskInfo>) mock(ReduceFunction.class);
ReducingStateDescriptor<TaskInfo> descr =
new ReducingStateDescriptor<>("name", reducer, TaskInfo.class);
context.getReducingState(descr);
StateDescriptor<?, ?> descrIntercepted = (StateDescriptor<?, ?>) descriptorCapture.get();
TypeSerializer<?> serializer = descrIntercepted.getSerializer();
// check that the Path class is really registered, i.e., the execution config was applied
assertTrue(serializer instanceof KryoSerializer);
assertTrue(((KryoSerializer<?>) serializer).getKryo().getRegistration(Path.class).getId() > 0);
}
示例11: testAggregatingStateInstantiation
import org.apache.flink.api.common.ExecutionConfig; //导入方法依赖的package包/类
@Test
public void testAggregatingStateInstantiation() throws Exception {
final ExecutionConfig config = new ExecutionConfig();
config.registerKryoType(Path.class);
final AtomicReference<Object> descriptorCapture = new AtomicReference<>();
StreamingRuntimeContext context = new StreamingRuntimeContext(
createDescriptorCapturingMockOp(descriptorCapture, config),
createMockEnvironment(),
Collections.<String, Accumulator<?, ?>>emptyMap());
@SuppressWarnings("unchecked")
AggregateFunction<String, TaskInfo, String> aggregate = (AggregateFunction<String, TaskInfo, String>) mock(AggregateFunction.class);
AggregatingStateDescriptor<String, TaskInfo, String> descr =
new AggregatingStateDescriptor<>("name", aggregate, TaskInfo.class);
context.getAggregatingState(descr);
AggregatingStateDescriptor<?, ?, ?> descrIntercepted = (AggregatingStateDescriptor<?, ?, ?>) descriptorCapture.get();
TypeSerializer<?> serializer = descrIntercepted.getSerializer();
// check that the Path class is really registered, i.e., the execution config was applied
assertTrue(serializer instanceof KryoSerializer);
assertTrue(((KryoSerializer<?>) serializer).getKryo().getRegistration(Path.class).getId() > 0);
}
示例12: testFoldingStateInstantiation
import org.apache.flink.api.common.ExecutionConfig; //导入方法依赖的package包/类
@Test
public void testFoldingStateInstantiation() throws Exception {
final ExecutionConfig config = new ExecutionConfig();
config.registerKryoType(Path.class);
final AtomicReference<Object> descriptorCapture = new AtomicReference<>();
StreamingRuntimeContext context = new StreamingRuntimeContext(
createDescriptorCapturingMockOp(descriptorCapture, config),
createMockEnvironment(),
Collections.<String, Accumulator<?, ?>>emptyMap());
@SuppressWarnings("unchecked")
FoldFunction<String, TaskInfo> folder = (FoldFunction<String, TaskInfo>) mock(FoldFunction.class);
FoldingStateDescriptor<String, TaskInfo> descr =
new FoldingStateDescriptor<>("name", null, folder, TaskInfo.class);
context.getFoldingState(descr);
FoldingStateDescriptor<?, ?> descrIntercepted = (FoldingStateDescriptor<?, ?>) descriptorCapture.get();
TypeSerializer<?> serializer = descrIntercepted.getSerializer();
// check that the Path class is really registered, i.e., the execution config was applied
assertTrue(serializer instanceof KryoSerializer);
assertTrue(((KryoSerializer<?>) serializer).getKryo().getRegistration(Path.class).getId() > 0);
}
示例13: testOutputBufferedBeingClearedInCaseOfException
import org.apache.flink.api.common.ExecutionConfig; //导入方法依赖的package包/类
/**
* Tests that the kryo output buffer is cleared in case of an exception. Flink uses the
* EOFException to signal that a buffer is full. In such a case, the record which was tried
* to be written will be rewritten. Therefore, eventually buffered data of this record has
* to be cleared.
*/
@Test
public void testOutputBufferedBeingClearedInCaseOfException() throws Exception {
ExecutionConfig executionConfig = new ExecutionConfig();
executionConfig.registerTypeWithKryoSerializer(TestRecord.class, new TestRecordSerializer());
executionConfig.registerKryoType(TestRecord.class);
KryoSerializer<TestRecord> kryoSerializer = new KryoSerializer<TestRecord>(
TestRecord.class,
executionConfig);
int size = 94;
int bufferSize = 150;
TestRecord testRecord = new TestRecord(size);
TestDataOutputView target = new TestDataOutputView(bufferSize);
kryoSerializer.serialize(testRecord, target);
try {
kryoSerializer.serialize(testRecord, target);
Assert.fail("Expected an EOFException.");
} catch(EOFException eofException) {
// expected exception
// now the Kryo Output should have been cleared
}
TestRecord actualRecord = kryoSerializer.deserialize(
new DataInputViewStreamWrapper(new ByteArrayInputStream(target.getBuffer())));
Assert.assertEquals(testRecord, actualRecord);
target.clear();
// if the kryo output has been cleared then we can serialize our test record into the target
// because the target buffer 150 bytes can host one TestRecord (total serialization size 100)
kryoSerializer.serialize(testRecord, target);
byte[] buffer = target.getBuffer();
int counter = 0;
for (int i = 0; i < buffer.length; i++) {
if(buffer[i] == 42) {
counter++;
}
}
Assert.assertEquals(size, counter);
}