本文整理汇总了Java中org.apache.tajo.datum.ProtobufDatumFactory类的典型用法代码示例。如果您正苦于以下问题:Java ProtobufDatumFactory类的具体用法?Java ProtobufDatumFactory怎么用?Java ProtobufDatumFactory使用的例子?那么恭喜您, 这里精选的类代码示例或许可以为您提供帮助。
ProtobufDatumFactory类属于org.apache.tajo.datum包,在下文中一共展示了ProtobufDatumFactory类的15个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Java代码示例。
示例1: testCreate
import org.apache.tajo.datum.ProtobufDatumFactory; //导入依赖的package包/类
@Test
public void testCreate() throws IOException, NoSuchMethodException, ClassNotFoundException {
TajoDataTypes.DataType.Builder builder = TajoDataTypes.DataType.newBuilder();
builder.setType(TajoDataTypes.Type.PROTOBUF);
builder.setCode(TajoIdProtos.QueryIdProto.class.getName());
ProtobufDatumFactory factory = ProtobufDatumFactory.get(builder.build());
TajoIdProtos.QueryIdProto.Builder queryIdBuilder = factory.newBuilder();
queryIdBuilder.setId(String.valueOf(System.currentTimeMillis()));
queryIdBuilder.setSeq(1);
TajoIdProtos.QueryIdProto queryId = queryIdBuilder.build();
ProtobufDatum datum = factory.createDatum(queryId);
ProtobufJsonFormat formatter = ProtobufJsonFormat.getInstance();
String json = formatter.printToString(datum.get());
TajoIdProtos.QueryIdProto.Builder fromJson = factory.newBuilder();
formatter.merge(TextUtils.toInputStream(json), fromJson);
assertEquals(queryId, fromJson.build());
}
示例2: addBinary
import org.apache.tajo.datum.ProtobufDatumFactory; //导入依赖的package包/类
@Override
final public void addBinary(Binary value) {
try {
ProtobufDatumFactory factory =
ProtobufDatumFactory.get(dataType.getCode());
Message.Builder builder = factory.newBuilder();
builder.mergeFrom(value.getBytes());
parent.add(factory.createDatum(builder));
} catch (InvalidProtocolBufferException e) {
throw new RuntimeException(e);
}
}
示例3: testVariousTypes
import org.apache.tajo.datum.ProtobufDatumFactory; //导入依赖的package包/类
@Test
public void testVariousTypes() throws IOException {
SchemaBuilder schemaBld = SchemaBuilder.builder()
.add("col1", Type.BOOLEAN)
.add("col2", CatalogUtil.newDataTypeWithLen(Type.CHAR, 7))
.add("col3", Type.INT2)
.add("col4", Type.INT4)
.add("col5", Type.INT8)
.add("col6", Type.FLOAT4)
.add("col7", Type.FLOAT8)
.add("col8", Type.TEXT)
.add("col9", Type.BLOB);
if (dateTypeSupport()) {
schemaBld.add("col10", Type.DATE);
}
if (protoTypeSupport()) {
schemaBld.add("col11", CatalogUtil.newDataType(Type.PROTOBUF, TajoIdProtos.QueryIdProto.class.getName()));
}
Schema schema = schemaBld.build();
TableMeta meta = CatalogUtil.newTableMeta(dataFormat, conf);
if (dataFormat.equalsIgnoreCase(BuiltinStorages.AVRO)) {
String path = JavaResourceUtil.getResourceURL("dataset/testVariousTypes.avsc").toString();
meta.putProperty(StorageConstants.AVRO_SCHEMA_URL, path);
}
FileTablespace sm = TablespaceManager.getLocalFs();
Path tablePath = new Path(testDir, "testVariousTypes.data");
Appender appender = sm.getAppender(meta, schema, tablePath);
appender.init();
QueryId queryid = new QueryId("12345", 5);
VTuple tuple = new VTuple(9 + (dateTypeSupport() ? 1 : 0) + (protoTypeSupport() ? 1 : 0));
tuple.put(new Datum[] {
DatumFactory.createBool(true),
DatumFactory.createChar("hyunsik"),
DatumFactory.createInt2((short) 17),
DatumFactory.createInt4(59),
DatumFactory.createInt8(23L),
DatumFactory.createFloat4(77.9f),
DatumFactory.createFloat8(271.9f),
DatumFactory.createText("hyunsik"),
DatumFactory.createBlob("hyunsik".getBytes()),
});
short currentIdx = 9;
if (dateTypeSupport()) {
tuple.put(currentIdx, DatumFactory.createDate(2016, 6, 28));
currentIdx++;
}
if (protoTypeSupport()) {
tuple.put(currentIdx, ProtobufDatumFactory.createDatum(queryid.getProto()));
}
appender.addTuple(tuple);
appender.flush();
appender.close();
FileStatus status = fs.getFileStatus(tablePath);
FileFragment fragment = new FileFragment("table", tablePath, 0, status.getLen());
Scanner scanner = sm.getScanner(meta, schema, fragment, null);
scanner.init();
Tuple retrieved;
while ((retrieved = scanner.next()) != null) {
for (int i = 0; i < tuple.size(); i++) {
assertEquals(tuple.get(i), retrieved.asDatum(i));
}
}
scanner.close();
}
示例4: testRCFileTextSerializeDeserialize
import org.apache.tajo.datum.ProtobufDatumFactory; //导入依赖的package包/类
@Test
public void testRCFileTextSerializeDeserialize() throws IOException {
if(!dataFormat.equalsIgnoreCase(BuiltinStorages.RCFILE)) return;
Schema schema = SchemaBuilder.builder()
.add("col1", Type.BOOLEAN)
.add("col2", CatalogUtil.newDataTypeWithLen(Type.CHAR, 7))
.add("col3", Type.INT2)
.add("col4", Type.INT4)
.add("col5", Type.INT8)
.add("col6", Type.FLOAT4)
.add("col7", Type.FLOAT8)
.add("col8", Type.TEXT)
.add("col9", Type.BLOB)
.add("col10", CatalogUtil.newDataType(Type.PROTOBUF, TajoIdProtos.QueryIdProto.class.getName()))
.build();
TableMeta meta = CatalogUtil.newTableMeta(dataFormat, conf);
meta.putProperty(StorageConstants.CSVFILE_SERDE, TextSerializerDeserializer.class.getName());
Path tablePath = new Path(testDir, "testRCFileTextSerializeDeserialize.data");
FileTablespace sm = TablespaceManager.getLocalFs();
Appender appender = sm.getAppender(meta, schema, tablePath);
appender.enableStats();
appender.init();
QueryId queryid = new QueryId("12345", 5);
ProtobufDatumFactory factory = ProtobufDatumFactory.get(TajoIdProtos.QueryIdProto.class.getName());
VTuple tuple = new VTuple(new Datum[] {
DatumFactory.createBool(true),
DatumFactory.createChar("jinho"),
DatumFactory.createInt2((short) 17),
DatumFactory.createInt4(59),
DatumFactory.createInt8(23l),
DatumFactory.createFloat4(77.9f),
DatumFactory.createFloat8(271.9f),
DatumFactory.createText("jinho"),
DatumFactory.createBlob("hyunsik babo".getBytes()),
factory.createDatum(queryid.getProto())
});
appender.addTuple(tuple);
appender.flush();
appender.close();
FileStatus status = fs.getFileStatus(tablePath);
assertEquals(appender.getStats().getNumBytes().longValue(), status.getLen());
FileFragment fragment = new FileFragment("table", tablePath, 0, status.getLen());
Scanner scanner = TablespaceManager.getLocalFs().getScanner(meta, schema, fragment, null);
scanner.init();
Tuple retrieved;
while ((retrieved=scanner.next()) != null) {
for (int i = 0; i < tuple.size(); i++) {
assertEquals(tuple.get(i), retrieved.asDatum(i));
}
}
scanner.close();
assertEquals(appender.getStats().getNumBytes().longValue(), scanner.getInputStats().getNumBytes().longValue());
assertEquals(appender.getStats().getNumRows().longValue(), scanner.getInputStats().getNumRows().longValue());
}
示例5: testRCFileBinarySerializeDeserialize
import org.apache.tajo.datum.ProtobufDatumFactory; //导入依赖的package包/类
@Test
public void testRCFileBinarySerializeDeserialize() throws IOException {
if(!dataFormat.equalsIgnoreCase(BuiltinStorages.RCFILE)) return;
Schema schema = SchemaBuilder.builder()
.add("col1", Type.BOOLEAN)
.add("col2", CatalogUtil.newDataTypeWithLen(Type.CHAR, 7))
.add("col3", Type.INT2)
.add("col4", Type.INT4)
.add("col5", Type.INT8)
.add("col6", Type.FLOAT4)
.add("col7", Type.FLOAT8)
.add("col8", Type.TEXT)
.add("col9", Type.BLOB)
.add("col10", CatalogUtil.newDataType(Type.PROTOBUF, TajoIdProtos.QueryIdProto.class.getName()))
.build();
TableMeta meta = CatalogUtil.newTableMeta(dataFormat, conf);
meta.putProperty(StorageConstants.RCFILE_SERDE, BinarySerializerDeserializer.class.getName());
Path tablePath = new Path(testDir, "testRCFileBinarySerializeDeserialize.data");
FileTablespace sm = TablespaceManager.getLocalFs();
Appender appender = sm.getAppender(meta, schema, tablePath);
appender.enableStats();
appender.init();
QueryId queryid = new QueryId("12345", 5);
ProtobufDatumFactory factory = ProtobufDatumFactory.get(TajoIdProtos.QueryIdProto.class.getName());
VTuple tuple = new VTuple(new Datum[] {
DatumFactory.createBool(true),
DatumFactory.createChar("jinho"),
DatumFactory.createInt2((short) 17),
DatumFactory.createInt4(59),
DatumFactory.createInt8(23l),
DatumFactory.createFloat4(77.9f),
DatumFactory.createFloat8(271.9f),
DatumFactory.createText("jinho"),
DatumFactory.createBlob("hyunsik babo".getBytes()),
factory.createDatum(queryid.getProto())
});
appender.addTuple(tuple);
appender.flush();
appender.close();
FileStatus status = fs.getFileStatus(tablePath);
assertEquals(appender.getStats().getNumBytes().longValue(), status.getLen());
FileFragment fragment = new FileFragment("table", tablePath, 0, status.getLen());
Scanner scanner = TablespaceManager.getLocalFs().getScanner(meta, schema, fragment, null);
scanner.init();
Tuple retrieved;
while ((retrieved=scanner.next()) != null) {
for (int i = 0; i < tuple.size(); i++) {
assertEquals(tuple.get(i), retrieved.asDatum(i));
}
}
scanner.close();
assertEquals(appender.getStats().getNumBytes().longValue(), scanner.getInputStats().getNumBytes().longValue());
assertEquals(appender.getStats().getNumRows().longValue(), scanner.getInputStats().getNumRows().longValue());
}
示例6: testSequenceFileTextSerializeDeserialize
import org.apache.tajo.datum.ProtobufDatumFactory; //导入依赖的package包/类
@Test
public void testSequenceFileTextSerializeDeserialize() throws IOException {
if(!dataFormat.equalsIgnoreCase(BuiltinStorages.SEQUENCE_FILE)) return;
Schema schema = SchemaBuilder.builder()
.add("col1", Type.BOOLEAN)
.add("col2", CatalogUtil.newDataTypeWithLen(Type.CHAR, 7))
.add("col3", Type.INT2)
.add("col4", Type.INT4)
.add("col5", Type.INT8)
.add("col6", Type.FLOAT4)
.add("col7", Type.FLOAT8)
.add("col8", Type.TEXT)
.add("col9", Type.BLOB)
.add("col10", CatalogUtil.newDataType(Type.PROTOBUF, TajoIdProtos.QueryIdProto.class.getName())).build();
TableMeta meta = CatalogUtil.newTableMeta(dataFormat, conf);
meta.putProperty(StorageConstants.SEQUENCEFILE_SERDE, TextSerializerDeserializer.class.getName());
Path tablePath = new Path(testDir, "testSequenceFileTextSerializeDeserialize.data");
FileTablespace sm = TablespaceManager.getLocalFs();
Appender appender = sm.getAppender(meta, schema, tablePath);
appender.enableStats();
appender.init();
QueryId queryid = new QueryId("12345", 5);
VTuple tuple = new VTuple(new Datum[] {
DatumFactory.createBool(true),
DatumFactory.createChar("jinho"),
DatumFactory.createInt2((short) 17),
DatumFactory.createInt4(59),
DatumFactory.createInt8(23l),
DatumFactory.createFloat4(77.9f),
DatumFactory.createFloat8(271.9f),
DatumFactory.createText("jinho"),
DatumFactory.createBlob("hyunsik babo".getBytes()),
ProtobufDatumFactory.createDatum(queryid.getProto())
});
appender.addTuple(tuple);
appender.flush();
appender.close();
FileStatus status = fs.getFileStatus(tablePath);
assertEquals(appender.getStats().getNumBytes().longValue(), status.getLen());
FileFragment fragment = new FileFragment("table", tablePath, 0, status.getLen());
Scanner scanner = TablespaceManager.getLocalFs().getScanner(meta, schema, fragment, null);
scanner.init();
assertTrue(scanner instanceof SequenceFileScanner);
Writable key = ((SequenceFileScanner) scanner).getKey();
assertEquals(key.getClass().getCanonicalName(), LongWritable.class.getCanonicalName());
Tuple retrieved;
while ((retrieved=scanner.next()) != null) {
for (int i = 0; i < tuple.size(); i++) {
assertEquals(tuple.get(i), retrieved.asDatum(i));
}
}
scanner.close();
assertEquals(appender.getStats().getNumBytes().longValue(), scanner.getInputStats().getNumBytes().longValue());
assertEquals(appender.getStats().getNumRows().longValue(), scanner.getInputStats().getNumRows().longValue());
}
示例7: testSequenceFileTextSerializeDeserializeWithDeprecatedProperties
import org.apache.tajo.datum.ProtobufDatumFactory; //导入依赖的package包/类
@Test
public void testSequenceFileTextSerializeDeserializeWithDeprecatedProperties() throws IOException {
if(!dataFormat.equalsIgnoreCase(BuiltinStorages.SEQUENCE_FILE)) return;
Schema schema = SchemaBuilder.builder()
.add("col1", Type.BOOLEAN)
.add("col2", CatalogUtil.newDataTypeWithLen(Type.CHAR, 7))
.add("col3", Type.INT2)
.add("col4", Type.INT4)
.add("col5", Type.INT8)
.add("col6", Type.FLOAT4)
.add("col7", Type.FLOAT8)
.add("col8", Type.TEXT)
.add("col9", Type.BLOB)
.add("col10", CatalogUtil.newDataType(Type.PROTOBUF, TajoIdProtos.QueryIdProto.class.getName())).build();
TableMeta meta = CatalogUtil.newTableMeta(dataFormat, conf);
meta.putProperty(StorageConstants.SEQUENCEFILE_SERDE, TextSerializerDeserializer.class.getName());
meta.putProperty(StorageConstants.SEQUENCEFILE_DELIMITER, "\u0001");
meta.putProperty(StorageConstants.SEQUENCEFILE_NULL, "\\");
Path tablePath = new Path(testDir, "testSequenceFileTextSerializeDeserializeWithDeprecatedProperties.data");
FileTablespace sm = TablespaceManager.getLocalFs();
Appender appender = sm.getAppender(meta, schema, tablePath);
appender.enableStats();
appender.init();
QueryId queryid = new QueryId("12345", 5);
VTuple tuple = new VTuple(new Datum[] {
DatumFactory.createBool(true),
DatumFactory.createChar("jinho"),
DatumFactory.createInt2((short) 17),
DatumFactory.createInt4(59),
DatumFactory.createInt8(23l),
DatumFactory.createFloat4(77.9f),
DatumFactory.createFloat8(271.9f),
DatumFactory.createText("jinho"),
DatumFactory.createBlob("hyunsik babo".getBytes()),
ProtobufDatumFactory.createDatum(queryid.getProto())
});
appender.addTuple(tuple);
appender.flush();
appender.close();
FileStatus status = fs.getFileStatus(tablePath);
assertEquals(appender.getStats().getNumBytes().longValue(), status.getLen());
FileFragment fragment = new FileFragment("table", tablePath, 0, status.getLen());
Scanner scanner = TablespaceManager.getLocalFs().getScanner(meta, schema, fragment, null);
scanner.init();
assertTrue(scanner instanceof SequenceFileScanner);
Writable key = ((SequenceFileScanner) scanner).getKey();
assertEquals(key.getClass().getCanonicalName(), LongWritable.class.getCanonicalName());
Tuple retrieved;
while ((retrieved=scanner.next()) != null) {
for (int i = 0; i < tuple.size(); i++) {
assertEquals(tuple.get(i), retrieved.asDatum(i));
}
}
scanner.close();
assertEquals(appender.getStats().getNumBytes().longValue(), scanner.getInputStats().getNumBytes().longValue());
assertEquals(appender.getStats().getNumRows().longValue(), scanner.getInputStats().getNumRows().longValue());
}
示例8: testSequenceFileTextSerializeDeserializeWithNonDeprecatedProperties
import org.apache.tajo.datum.ProtobufDatumFactory; //导入依赖的package包/类
@Test
public void testSequenceFileTextSerializeDeserializeWithNonDeprecatedProperties() throws IOException {
if(!dataFormat.equalsIgnoreCase(BuiltinStorages.SEQUENCE_FILE)) return;
Schema schema = SchemaBuilder.builder()
.add("col1", Type.BOOLEAN)
.add("col2", CatalogUtil.newDataTypeWithLen(Type.CHAR, 7))
.add("col3", Type.INT2)
.add("col4", Type.INT4)
.add("col5", Type.INT8)
.add("col6", Type.FLOAT4)
.add("col7", Type.FLOAT8)
.add("col8", Type.TEXT)
.add("col9", Type.BLOB)
.add("col10", CatalogUtil.newDataType(Type.PROTOBUF, TajoIdProtos.QueryIdProto.class.getName())).build();
TableMeta meta = CatalogUtil.newTableMeta(dataFormat, conf);
meta.putProperty(StorageConstants.SEQUENCEFILE_SERDE, TextSerializerDeserializer.class.getName());
meta.putProperty(StorageConstants.TEXT_DELIMITER, "\u0001");
meta.putProperty(StorageConstants.TEXT_NULL, "\\");
Path tablePath = new Path(testDir, "testSequenceFileTextSerializeDeserializeWithNonDeprecatedProperties.data");
FileTablespace sm = TablespaceManager.getLocalFs();
Appender appender = sm.getAppender(meta, schema, tablePath);
appender.enableStats();
appender.init();
QueryId queryid = new QueryId("12345", 5);
VTuple tuple = new VTuple(new Datum[] {
DatumFactory.createBool(true),
DatumFactory.createChar("jinho"),
DatumFactory.createInt2((short) 17),
DatumFactory.createInt4(59),
DatumFactory.createInt8(23l),
DatumFactory.createFloat4(77.9f),
DatumFactory.createFloat8(271.9f),
DatumFactory.createText("jinho"),
DatumFactory.createBlob("hyunsik babo".getBytes()),
ProtobufDatumFactory.createDatum(queryid.getProto())
});
appender.addTuple(tuple);
appender.flush();
appender.close();
FileStatus status = fs.getFileStatus(tablePath);
assertEquals(appender.getStats().getNumBytes().longValue(), status.getLen());
FileFragment fragment = new FileFragment("table", tablePath, 0, status.getLen());
Scanner scanner = TablespaceManager.getLocalFs().getScanner(meta, schema, fragment, null);
scanner.init();
assertTrue(scanner instanceof SequenceFileScanner);
Writable key = ((SequenceFileScanner) scanner).getKey();
assertEquals(key.getClass().getCanonicalName(), LongWritable.class.getCanonicalName());
Tuple retrieved;
while ((retrieved=scanner.next()) != null) {
for (int i = 0; i < tuple.size(); i++) {
assertEquals(tuple.get(i), retrieved.asDatum(i));
}
}
scanner.close();
assertEquals(appender.getStats().getNumBytes().longValue(), scanner.getInputStats().getNumBytes().longValue());
assertEquals(appender.getStats().getNumRows().longValue(), scanner.getInputStats().getNumRows().longValue());
}
示例9: testSequenceFileBinarySerializeDeserialize
import org.apache.tajo.datum.ProtobufDatumFactory; //导入依赖的package包/类
@Test
public void testSequenceFileBinarySerializeDeserialize() throws IOException {
if(!dataFormat.equalsIgnoreCase(BuiltinStorages.SEQUENCE_FILE)) return;
Schema schema = SchemaBuilder.builder()
.add("col1", Type.BOOLEAN)
.add("col2", CatalogUtil.newDataTypeWithLen(Type.CHAR, 7))
.add("col3", Type.INT2)
.add("col4", Type.INT4)
.add("col5", Type.INT8)
.add("col6", Type.FLOAT4)
.add("col7", Type.FLOAT8)
.add("col8", Type.TEXT)
.add("col9", Type.BLOB)
.add("col10", CatalogUtil.newDataType(Type.PROTOBUF, TajoIdProtos.QueryIdProto.class.getName()))
.build();
TableMeta meta = CatalogUtil.newTableMeta(dataFormat, conf);
meta.putProperty(StorageConstants.SEQUENCEFILE_SERDE, BinarySerializerDeserializer.class.getName());
Path tablePath = new Path(testDir, "testVariousTypes.data");
FileTablespace sm = TablespaceManager.getLocalFs();
Appender appender = sm.getAppender(meta, schema, tablePath);
appender.enableStats();
appender.init();
QueryId queryid = new QueryId("12345", 5);
VTuple tuple = new VTuple(new Datum[] {
DatumFactory.createBool(true),
DatumFactory.createChar("jinho"),
DatumFactory.createInt2((short) 17),
DatumFactory.createInt4(59),
DatumFactory.createInt8(23l),
DatumFactory.createFloat4(77.9f),
DatumFactory.createFloat8(271.9f),
DatumFactory.createText("jinho"),
DatumFactory.createBlob("hyunsik babo".getBytes()),
ProtobufDatumFactory.createDatum(queryid.getProto())
});
appender.addTuple(tuple);
appender.flush();
appender.close();
FileStatus status = fs.getFileStatus(tablePath);
assertEquals(appender.getStats().getNumBytes().longValue(), status.getLen());
FileFragment fragment = new FileFragment("table", tablePath, 0, status.getLen());
Scanner scanner = TablespaceManager.getLocalFs().getScanner(meta, schema, fragment, null);
scanner.init();
assertTrue(scanner instanceof SequenceFileScanner);
Writable key = ((SequenceFileScanner) scanner).getKey();
assertEquals(key.getClass().getCanonicalName(), BytesWritable.class.getCanonicalName());
Tuple retrieved;
while ((retrieved=scanner.next()) != null) {
for (int i = 0; i < tuple.size(); i++) {
assertEquals(tuple.get(i), retrieved.asDatum(i));
}
}
scanner.close();
assertEquals(appender.getStats().getNumBytes().longValue(), scanner.getInputStats().getNumBytes().longValue());
assertEquals(appender.getStats().getNumRows().longValue(), scanner.getInputStats().getNumRows().longValue());
}
示例10: testVariousTypes
import org.apache.tajo.datum.ProtobufDatumFactory; //导入依赖的package包/类
@Test
public void testVariousTypes() throws IOException {
Schema schema = new Schema();
schema.addColumn("col1", Type.BOOLEAN);
schema.addColumn("col2", Type.BIT);
schema.addColumn("col3", Type.CHAR, 7);
schema.addColumn("col4", Type.INT2);
schema.addColumn("col5", Type.INT4);
schema.addColumn("col6", Type.INT8);
schema.addColumn("col7", Type.FLOAT4);
schema.addColumn("col8", Type.FLOAT8);
schema.addColumn("col9", Type.TEXT);
schema.addColumn("col10", Type.BLOB);
schema.addColumn("col11", Type.INET4);
schema.addColumn("col12", Type.NULL_TYPE);
schema.addColumn("col13", CatalogUtil.newDataType(Type.PROTOBUF, TajoIdProtos.QueryIdProto.class.getName()));
Options options = new Options();
TableMeta meta = CatalogUtil.newTableMeta(storeType, options);
Path tablePath = new Path(testDir, "testVariousTypes.data");
Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(meta, schema, tablePath);
appender.init();
QueryId queryid = new QueryId("12345", 5);
ProtobufDatumFactory factory = ProtobufDatumFactory.get(TajoIdProtos.QueryIdProto.class.getName());
Tuple tuple = new VTuple(13);
tuple.put(new Datum[] {
DatumFactory.createBool(true),
DatumFactory.createBit((byte) 0x99),
DatumFactory.createChar("hyunsik"),
DatumFactory.createInt2((short) 17),
DatumFactory.createInt4(59),
DatumFactory.createInt8(23l),
DatumFactory.createFloat4(77.9f),
DatumFactory.createFloat8(271.9f),
DatumFactory.createText("hyunsik"),
DatumFactory.createBlob("hyunsik".getBytes()),
DatumFactory.createInet4("192.168.0.1"),
NullDatum.get(),
factory.createDatum(queryid.getProto())
});
appender.addTuple(tuple);
appender.flush();
appender.close();
FileStatus status = fs.getFileStatus(tablePath);
FileFragment fragment = new FileFragment("table", tablePath, 0, status.getLen());
Scanner scanner = StorageManagerFactory.getStorageManager(conf).getScanner(meta, schema, fragment);
scanner.init();
Tuple retrieved;
while ((retrieved=scanner.next()) != null) {
for (int i = 0; i < tuple.size(); i++) {
assertEquals(tuple.get(i), retrieved.get(i));
}
}
scanner.close();
}
示例11: testRCFileTextSerializeDeserialize
import org.apache.tajo.datum.ProtobufDatumFactory; //导入依赖的package包/类
@Test
public void testRCFileTextSerializeDeserialize() throws IOException {
if(storeType != StoreType.RCFILE) return;
Schema schema = new Schema();
schema.addColumn("col1", Type.BOOLEAN);
schema.addColumn("col2", Type.BIT);
schema.addColumn("col3", Type.CHAR, 7);
schema.addColumn("col4", Type.INT2);
schema.addColumn("col5", Type.INT4);
schema.addColumn("col6", Type.INT8);
schema.addColumn("col7", Type.FLOAT4);
schema.addColumn("col8", Type.FLOAT8);
schema.addColumn("col9", Type.TEXT);
schema.addColumn("col10", Type.BLOB);
schema.addColumn("col11", Type.INET4);
schema.addColumn("col12", Type.NULL_TYPE);
schema.addColumn("col13", CatalogUtil.newDataType(Type.PROTOBUF, TajoIdProtos.QueryIdProto.class.getName()));
Options options = new Options();
TableMeta meta = CatalogUtil.newTableMeta(storeType, options);
meta.putOption(CatalogConstants.CSVFILE_SERDE, TextSerializerDeserializer.class.getName());
Path tablePath = new Path(testDir, "testVariousTypes.data");
Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(meta, schema, tablePath);
appender.enableStats();
appender.init();
QueryId queryid = new QueryId("12345", 5);
ProtobufDatumFactory factory = ProtobufDatumFactory.get(TajoIdProtos.QueryIdProto.class.getName());
Tuple tuple = new VTuple(13);
tuple.put(new Datum[] {
DatumFactory.createBool(true),
DatumFactory.createBit((byte) 0x99),
DatumFactory.createChar("jinho"),
DatumFactory.createInt2((short) 17),
DatumFactory.createInt4(59),
DatumFactory.createInt8(23l),
DatumFactory.createFloat4(77.9f),
DatumFactory.createFloat8(271.9f),
DatumFactory.createText("jinho"),
DatumFactory.createBlob("hyunsik babo".getBytes()),
DatumFactory.createInet4("192.168.0.1"),
NullDatum.get(),
factory.createDatum(queryid.getProto())
});
appender.addTuple(tuple);
appender.flush();
appender.close();
FileStatus status = fs.getFileStatus(tablePath);
assertEquals(appender.getStats().getNumBytes().longValue(), status.getLen());
FileFragment fragment = new FileFragment("table", tablePath, 0, status.getLen());
Scanner scanner = StorageManagerFactory.getStorageManager(conf).getScanner(meta, schema, fragment);
scanner.init();
Tuple retrieved;
while ((retrieved=scanner.next()) != null) {
for (int i = 0; i < tuple.size(); i++) {
assertEquals(tuple.get(i), retrieved.get(i));
}
}
scanner.close();
assertEquals(appender.getStats().getNumBytes().longValue(), scanner.getInputStats().getNumBytes().longValue());
assertEquals(appender.getStats().getNumRows().longValue(), scanner.getInputStats().getNumRows().longValue());
}
示例12: testRCFileBinarySerializeDeserialize
import org.apache.tajo.datum.ProtobufDatumFactory; //导入依赖的package包/类
@Test
public void testRCFileBinarySerializeDeserialize() throws IOException {
if(storeType != StoreType.RCFILE) return;
Schema schema = new Schema();
schema.addColumn("col1", Type.BOOLEAN);
schema.addColumn("col2", Type.BIT);
schema.addColumn("col3", Type.CHAR, 7);
schema.addColumn("col4", Type.INT2);
schema.addColumn("col5", Type.INT4);
schema.addColumn("col6", Type.INT8);
schema.addColumn("col7", Type.FLOAT4);
schema.addColumn("col8", Type.FLOAT8);
schema.addColumn("col9", Type.TEXT);
schema.addColumn("col10", Type.BLOB);
schema.addColumn("col11", Type.INET4);
schema.addColumn("col12", Type.NULL_TYPE);
schema.addColumn("col13", CatalogUtil.newDataType(Type.PROTOBUF, TajoIdProtos.QueryIdProto.class.getName()));
Options options = new Options();
TableMeta meta = CatalogUtil.newTableMeta(storeType, options);
meta.putOption(CatalogConstants.RCFILE_SERDE, BinarySerializerDeserializer.class.getName());
Path tablePath = new Path(testDir, "testVariousTypes.data");
Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(meta, schema, tablePath);
appender.enableStats();
appender.init();
QueryId queryid = new QueryId("12345", 5);
ProtobufDatumFactory factory = ProtobufDatumFactory.get(TajoIdProtos.QueryIdProto.class.getName());
Tuple tuple = new VTuple(13);
tuple.put(new Datum[] {
DatumFactory.createBool(true),
DatumFactory.createBit((byte) 0x99),
DatumFactory.createChar("jinho"),
DatumFactory.createInt2((short) 17),
DatumFactory.createInt4(59),
DatumFactory.createInt8(23l),
DatumFactory.createFloat4(77.9f),
DatumFactory.createFloat8(271.9f),
DatumFactory.createText("jinho"),
DatumFactory.createBlob("hyunsik babo".getBytes()),
DatumFactory.createInet4("192.168.0.1"),
NullDatum.get(),
factory.createDatum(queryid.getProto())
});
appender.addTuple(tuple);
appender.flush();
appender.close();
FileStatus status = fs.getFileStatus(tablePath);
assertEquals(appender.getStats().getNumBytes().longValue(), status.getLen());
FileFragment fragment = new FileFragment("table", tablePath, 0, status.getLen());
Scanner scanner = StorageManagerFactory.getStorageManager(conf).getScanner(meta, schema, fragment);
scanner.init();
Tuple retrieved;
while ((retrieved=scanner.next()) != null) {
for (int i = 0; i < tuple.size(); i++) {
assertEquals(tuple.get(i), retrieved.get(i));
}
}
scanner.close();
assertEquals(appender.getStats().getNumBytes().longValue(), scanner.getInputStats().getNumBytes().longValue());
assertEquals(appender.getStats().getNumRows().longValue(), scanner.getInputStats().getNumRows().longValue());
}
示例13: testVariousTypes
import org.apache.tajo.datum.ProtobufDatumFactory; //导入依赖的package包/类
@Test
public void testVariousTypes() throws IOException {
Schema schema = new Schema();
schema.addColumn("col1", Type.BOOLEAN);
schema.addColumn("col2", Type.BIT);
schema.addColumn("col3", Type.CHAR, 7);
schema.addColumn("col4", Type.INT2);
schema.addColumn("col5", Type.INT4);
schema.addColumn("col6", Type.INT8);
schema.addColumn("col7", Type.FLOAT4);
schema.addColumn("col8", Type.FLOAT8);
schema.addColumn("col9", Type.TEXT);
schema.addColumn("col10", Type.BLOB);
schema.addColumn("col11", Type.INET4);
schema.addColumn("col12", Type.NULL_TYPE);
schema.addColumn("col13", CatalogUtil.newDataType(Type.PROTOBUF, TajoIdProtos.QueryIdProto.class.getName()));
Options options = new Options();
TableMeta meta = CatalogUtil.newTableMeta(storeType, options);
meta.setOptions(StorageUtil.newPhysicalProperties(storeType));
if (storeType == StoreType.AVRO) {
String path = FileUtil.getResourcePath("testVariousTypes.avsc").toString();
meta.putOption(StorageConstants.AVRO_SCHEMA_URL, path);
}
Path tablePath = new Path(testDir, "testVariousTypes.data");
Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(meta, schema, tablePath);
appender.init();
QueryId queryid = new QueryId("12345", 5);
ProtobufDatumFactory factory = ProtobufDatumFactory.get(TajoIdProtos.QueryIdProto.class.getName());
Tuple tuple = new VTuple(13);
tuple.put(new Datum[] {
DatumFactory.createBool(true),
DatumFactory.createBit((byte) 0x99),
DatumFactory.createChar("hyunsik"),
DatumFactory.createInt2((short) 17),
DatumFactory.createInt4(59),
DatumFactory.createInt8(23l),
DatumFactory.createFloat4(77.9f),
DatumFactory.createFloat8(271.9f),
DatumFactory.createText("hyunsik"),
DatumFactory.createBlob("hyunsik".getBytes()),
DatumFactory.createInet4("192.168.0.1"),
NullDatum.get(),
factory.createDatum(queryid.getProto())
});
appender.addTuple(tuple);
appender.flush();
appender.close();
FileStatus status = fs.getFileStatus(tablePath);
FileFragment fragment = new FileFragment("table", tablePath, 0, status.getLen());
Scanner scanner = StorageManagerFactory.getStorageManager(conf).getScanner(meta, schema, fragment);
scanner.init();
Tuple retrieved;
while ((retrieved = scanner.next()) != null) {
for (int i = 0; i < tuple.size(); i++) {
assertEquals(tuple.get(i), retrieved.get(i));
}
}
scanner.close();
}
示例14: testRCFileTextSerializeDeserialize
import org.apache.tajo.datum.ProtobufDatumFactory; //导入依赖的package包/类
@Test
public void testRCFileTextSerializeDeserialize() throws IOException {
if(storeType != StoreType.RCFILE) return;
Schema schema = new Schema();
schema.addColumn("col1", Type.BOOLEAN);
schema.addColumn("col2", Type.BIT);
schema.addColumn("col3", Type.CHAR, 7);
schema.addColumn("col4", Type.INT2);
schema.addColumn("col5", Type.INT4);
schema.addColumn("col6", Type.INT8);
schema.addColumn("col7", Type.FLOAT4);
schema.addColumn("col8", Type.FLOAT8);
schema.addColumn("col9", Type.TEXT);
schema.addColumn("col10", Type.BLOB);
schema.addColumn("col11", Type.INET4);
schema.addColumn("col12", Type.NULL_TYPE);
schema.addColumn("col13", CatalogUtil.newDataType(Type.PROTOBUF, TajoIdProtos.QueryIdProto.class.getName()));
Options options = new Options();
TableMeta meta = CatalogUtil.newTableMeta(storeType, options);
meta.putOption(StorageConstants.CSVFILE_SERDE, TextSerializerDeserializer.class.getName());
Path tablePath = new Path(testDir, "testVariousTypes.data");
Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(meta, schema, tablePath);
appender.enableStats();
appender.init();
QueryId queryid = new QueryId("12345", 5);
ProtobufDatumFactory factory = ProtobufDatumFactory.get(TajoIdProtos.QueryIdProto.class.getName());
Tuple tuple = new VTuple(13);
tuple.put(new Datum[] {
DatumFactory.createBool(true),
DatumFactory.createBit((byte) 0x99),
DatumFactory.createChar("jinho"),
DatumFactory.createInt2((short) 17),
DatumFactory.createInt4(59),
DatumFactory.createInt8(23l),
DatumFactory.createFloat4(77.9f),
DatumFactory.createFloat8(271.9f),
DatumFactory.createText("jinho"),
DatumFactory.createBlob("hyunsik babo".getBytes()),
DatumFactory.createInet4("192.168.0.1"),
NullDatum.get(),
factory.createDatum(queryid.getProto())
});
appender.addTuple(tuple);
appender.flush();
appender.close();
FileStatus status = fs.getFileStatus(tablePath);
assertEquals(appender.getStats().getNumBytes().longValue(), status.getLen());
FileFragment fragment = new FileFragment("table", tablePath, 0, status.getLen());
Scanner scanner = StorageManagerFactory.getStorageManager(conf).getScanner(meta, schema, fragment);
scanner.init();
Tuple retrieved;
while ((retrieved=scanner.next()) != null) {
for (int i = 0; i < tuple.size(); i++) {
assertEquals(tuple.get(i), retrieved.get(i));
}
}
scanner.close();
assertEquals(appender.getStats().getNumBytes().longValue(), scanner.getInputStats().getNumBytes().longValue());
assertEquals(appender.getStats().getNumRows().longValue(), scanner.getInputStats().getNumRows().longValue());
}
示例15: testRCFileBinarySerializeDeserialize
import org.apache.tajo.datum.ProtobufDatumFactory; //导入依赖的package包/类
@Test
public void testRCFileBinarySerializeDeserialize() throws IOException {
if(storeType != StoreType.RCFILE) return;
Schema schema = new Schema();
schema.addColumn("col1", Type.BOOLEAN);
schema.addColumn("col2", Type.BIT);
schema.addColumn("col3", Type.CHAR, 7);
schema.addColumn("col4", Type.INT2);
schema.addColumn("col5", Type.INT4);
schema.addColumn("col6", Type.INT8);
schema.addColumn("col7", Type.FLOAT4);
schema.addColumn("col8", Type.FLOAT8);
schema.addColumn("col9", Type.TEXT);
schema.addColumn("col10", Type.BLOB);
schema.addColumn("col11", Type.INET4);
schema.addColumn("col12", Type.NULL_TYPE);
schema.addColumn("col13", CatalogUtil.newDataType(Type.PROTOBUF, TajoIdProtos.QueryIdProto.class.getName()));
Options options = new Options();
TableMeta meta = CatalogUtil.newTableMeta(storeType, options);
meta.putOption(StorageConstants.RCFILE_SERDE, BinarySerializerDeserializer.class.getName());
Path tablePath = new Path(testDir, "testVariousTypes.data");
Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(meta, schema, tablePath);
appender.enableStats();
appender.init();
QueryId queryid = new QueryId("12345", 5);
ProtobufDatumFactory factory = ProtobufDatumFactory.get(TajoIdProtos.QueryIdProto.class.getName());
Tuple tuple = new VTuple(13);
tuple.put(new Datum[] {
DatumFactory.createBool(true),
DatumFactory.createBit((byte) 0x99),
DatumFactory.createChar("jinho"),
DatumFactory.createInt2((short) 17),
DatumFactory.createInt4(59),
DatumFactory.createInt8(23l),
DatumFactory.createFloat4(77.9f),
DatumFactory.createFloat8(271.9f),
DatumFactory.createText("jinho"),
DatumFactory.createBlob("hyunsik babo".getBytes()),
DatumFactory.createInet4("192.168.0.1"),
NullDatum.get(),
factory.createDatum(queryid.getProto())
});
appender.addTuple(tuple);
appender.flush();
appender.close();
FileStatus status = fs.getFileStatus(tablePath);
assertEquals(appender.getStats().getNumBytes().longValue(), status.getLen());
FileFragment fragment = new FileFragment("table", tablePath, 0, status.getLen());
Scanner scanner = StorageManagerFactory.getStorageManager(conf).getScanner(meta, schema, fragment);
scanner.init();
Tuple retrieved;
while ((retrieved=scanner.next()) != null) {
for (int i = 0; i < tuple.size(); i++) {
assertEquals(tuple.get(i), retrieved.get(i));
}
}
scanner.close();
assertEquals(appender.getStats().getNumBytes().longValue(), scanner.getInputStats().getNumBytes().longValue());
assertEquals(appender.getStats().getNumRows().longValue(), scanner.getInputStats().getNumRows().longValue());
}