本文整理汇总了Java中org.apache.tajo.catalog.proto.CatalogProtos.StoreType.RCFILE属性的典型用法代码示例。如果您正苦于以下问题:Java StoreType.RCFILE属性的具体用法?Java StoreType.RCFILE怎么用?Java StoreType.RCFILE使用的例子?那么, 这里精选的属性代码示例或许可以为您提供帮助。您也可以进一步了解该属性所在类org.apache.tajo.catalog.proto.CatalogProtos.StoreType
的用法示例。
在下文中一共展示了StoreType.RCFILE属性的13个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Java代码示例。
示例1: TestStorages
public TestStorages(StoreType type, boolean splitable, boolean statsable) throws IOException {
this.storeType = type;
this.splitable = splitable;
this.statsable = statsable;
conf = new TajoConf();
conf.setBoolVar(ConfVars.STORAGE_MANAGER_VERSION_2, true);
if (storeType == StoreType.RCFILE) {
conf.setInt(RCFile.RECORD_INTERVAL_CONF_STR, 100);
}
testDir = CommonTestingUtil.getTestDir(TEST_PATH);
fs = testDir.getFileSystem(conf);
}
示例2: TestStorages
public TestStorages(StoreType type, boolean splitable, boolean statsable) throws IOException {
this.storeType = type;
this.splitable = splitable;
this.statsable = statsable;
conf = new TajoConf();
if (storeType == StoreType.RCFILE) {
conf.setInt(RCFile.RECORD_INTERVAL_CONF_STR, 100);
}
testDir = CommonTestingUtil.getTestDir(TEST_PATH);
fs = testDir.getFileSystem(conf);
}
示例3: testGzipCodecCompressionData
@Test
public void testGzipCodecCompressionData() throws IOException {
if (storeType == StoreType.RCFILE) {
if( ZlibFactory.isNativeZlibLoaded(conf)) {
storageCompressionTest(storeType, GzipCodec.class);
}
} else {
storageCompressionTest(storeType, GzipCodec.class);
}
}
示例4: TestStorages
public TestStorages(StoreType type, boolean splitable, boolean statsable) throws IOException {
this.storeType = type;
this.splitable = splitable;
this.statsable = statsable;
conf = new TajoConf();
if (storeType == StoreType.RCFILE) {
conf.setInt(RCFile.RECORD_INTERVAL_CONF_STR, 100);
}
testDir = CommonTestingUtil.getTestDir(TEST_PATH);
fs = testDir.getFileSystem(conf);
}
示例5: testGzipCodecCompressionData
@Test
public void testGzipCodecCompressionData() throws IOException {
if (storeType == StoreType.RCFILE) {
if( ZlibFactory.isNativeZlibLoaded(conf)) {
storageCompressionTest(storeType, GzipCodec.class);
}
} else if (storeType == StoreType.SEQUENCEFILE) {
if( ZlibFactory.isNativeZlibLoaded(conf)) {
storageCompressionTest(storeType, GzipCodec.class);
}
} else {
storageCompressionTest(storeType, GzipCodec.class);
}
}
示例6: testProjection
@Test
public void testProjection() throws IOException {
Schema schema = new Schema();
schema.addColumn("id", Type.INT4);
schema.addColumn("age", Type.INT8);
schema.addColumn("score", Type.FLOAT4);
TableMeta meta = CatalogUtil.newTableMeta(storeType);
Path tablePath = new Path(testDir, "testProjection.data");
Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(meta, schema, tablePath);
appender.init();
int tupleNum = 10000;
VTuple vTuple;
for(int i = 0; i < tupleNum; i++) {
vTuple = new VTuple(3);
vTuple.put(0, DatumFactory.createInt4(i + 1));
vTuple.put(1, DatumFactory.createInt8(i + 2));
vTuple.put(2, DatumFactory.createFloat4(i + 3));
appender.addTuple(vTuple);
}
appender.close();
FileStatus status = fs.getFileStatus(tablePath);
FileFragment fragment = new FileFragment("testReadAndWrite", tablePath, 0, status.getLen());
Schema target = new Schema();
target.addColumn("age", Type.INT8);
target.addColumn("score", Type.FLOAT4);
Scanner scanner = StorageManagerFactory.getStorageManager(conf).getScanner(meta, schema, fragment, target);
scanner.init();
int tupleCnt = 0;
Tuple tuple;
while ((tuple = scanner.next()) != null) {
if (storeType == StoreType.RCFILE || storeType == StoreType.TREVNI || storeType == StoreType.CSV) {
assertTrue(tuple.get(0) == null);
}
assertTrue(tupleCnt + 2 == tuple.get(1).asInt8());
assertTrue(tupleCnt + 3 == tuple.get(2).asFloat4());
tupleCnt++;
}
scanner.close();
assertEquals(tupleNum, tupleCnt);
}
示例7: testRCFileTextSerializeDeserialize
@Test
public void testRCFileTextSerializeDeserialize() throws IOException {
if(storeType != StoreType.RCFILE) return;
Schema schema = new Schema();
schema.addColumn("col1", Type.BOOLEAN);
schema.addColumn("col2", Type.BIT);
schema.addColumn("col3", Type.CHAR, 7);
schema.addColumn("col4", Type.INT2);
schema.addColumn("col5", Type.INT4);
schema.addColumn("col6", Type.INT8);
schema.addColumn("col7", Type.FLOAT4);
schema.addColumn("col8", Type.FLOAT8);
schema.addColumn("col9", Type.TEXT);
schema.addColumn("col10", Type.BLOB);
schema.addColumn("col11", Type.INET4);
schema.addColumn("col12", Type.NULL_TYPE);
schema.addColumn("col13", CatalogUtil.newDataType(Type.PROTOBUF, TajoIdProtos.QueryIdProto.class.getName()));
Options options = new Options();
TableMeta meta = CatalogUtil.newTableMeta(storeType, options);
meta.putOption(CatalogConstants.CSVFILE_SERDE, TextSerializerDeserializer.class.getName());
Path tablePath = new Path(testDir, "testVariousTypes.data");
Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(meta, schema, tablePath);
appender.enableStats();
appender.init();
QueryId queryid = new QueryId("12345", 5);
ProtobufDatumFactory factory = ProtobufDatumFactory.get(TajoIdProtos.QueryIdProto.class.getName());
Tuple tuple = new VTuple(13);
tuple.put(new Datum[] {
DatumFactory.createBool(true),
DatumFactory.createBit((byte) 0x99),
DatumFactory.createChar("jinho"),
DatumFactory.createInt2((short) 17),
DatumFactory.createInt4(59),
DatumFactory.createInt8(23l),
DatumFactory.createFloat4(77.9f),
DatumFactory.createFloat8(271.9f),
DatumFactory.createText("jinho"),
DatumFactory.createBlob("hyunsik babo".getBytes()),
DatumFactory.createInet4("192.168.0.1"),
NullDatum.get(),
factory.createDatum(queryid.getProto())
});
appender.addTuple(tuple);
appender.flush();
appender.close();
FileStatus status = fs.getFileStatus(tablePath);
assertEquals(appender.getStats().getNumBytes().longValue(), status.getLen());
FileFragment fragment = new FileFragment("table", tablePath, 0, status.getLen());
Scanner scanner = StorageManagerFactory.getStorageManager(conf).getScanner(meta, schema, fragment);
scanner.init();
Tuple retrieved;
while ((retrieved=scanner.next()) != null) {
for (int i = 0; i < tuple.size(); i++) {
assertEquals(tuple.get(i), retrieved.get(i));
}
}
scanner.close();
assertEquals(appender.getStats().getNumBytes().longValue(), scanner.getInputStats().getNumBytes().longValue());
assertEquals(appender.getStats().getNumRows().longValue(), scanner.getInputStats().getNumRows().longValue());
}
示例8: testRCFileBinarySerializeDeserialize
@Test
public void testRCFileBinarySerializeDeserialize() throws IOException {
if(storeType != StoreType.RCFILE) return;
Schema schema = new Schema();
schema.addColumn("col1", Type.BOOLEAN);
schema.addColumn("col2", Type.BIT);
schema.addColumn("col3", Type.CHAR, 7);
schema.addColumn("col4", Type.INT2);
schema.addColumn("col5", Type.INT4);
schema.addColumn("col6", Type.INT8);
schema.addColumn("col7", Type.FLOAT4);
schema.addColumn("col8", Type.FLOAT8);
schema.addColumn("col9", Type.TEXT);
schema.addColumn("col10", Type.BLOB);
schema.addColumn("col11", Type.INET4);
schema.addColumn("col12", Type.NULL_TYPE);
schema.addColumn("col13", CatalogUtil.newDataType(Type.PROTOBUF, TajoIdProtos.QueryIdProto.class.getName()));
Options options = new Options();
TableMeta meta = CatalogUtil.newTableMeta(storeType, options);
meta.putOption(CatalogConstants.RCFILE_SERDE, BinarySerializerDeserializer.class.getName());
Path tablePath = new Path(testDir, "testVariousTypes.data");
Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(meta, schema, tablePath);
appender.enableStats();
appender.init();
QueryId queryid = new QueryId("12345", 5);
ProtobufDatumFactory factory = ProtobufDatumFactory.get(TajoIdProtos.QueryIdProto.class.getName());
Tuple tuple = new VTuple(13);
tuple.put(new Datum[] {
DatumFactory.createBool(true),
DatumFactory.createBit((byte) 0x99),
DatumFactory.createChar("jinho"),
DatumFactory.createInt2((short) 17),
DatumFactory.createInt4(59),
DatumFactory.createInt8(23l),
DatumFactory.createFloat4(77.9f),
DatumFactory.createFloat8(271.9f),
DatumFactory.createText("jinho"),
DatumFactory.createBlob("hyunsik babo".getBytes()),
DatumFactory.createInet4("192.168.0.1"),
NullDatum.get(),
factory.createDatum(queryid.getProto())
});
appender.addTuple(tuple);
appender.flush();
appender.close();
FileStatus status = fs.getFileStatus(tablePath);
assertEquals(appender.getStats().getNumBytes().longValue(), status.getLen());
FileFragment fragment = new FileFragment("table", tablePath, 0, status.getLen());
Scanner scanner = StorageManagerFactory.getStorageManager(conf).getScanner(meta, schema, fragment);
scanner.init();
Tuple retrieved;
while ((retrieved=scanner.next()) != null) {
for (int i = 0; i < tuple.size(); i++) {
assertEquals(tuple.get(i), retrieved.get(i));
}
}
scanner.close();
assertEquals(appender.getStats().getNumBytes().longValue(), scanner.getInputStats().getNumBytes().longValue());
assertEquals(appender.getStats().getNumRows().longValue(), scanner.getInputStats().getNumRows().longValue());
}
示例9: testProjection
@Test
public void testProjection() throws IOException {
Schema schema = new Schema();
schema.addColumn("id", Type.INT4);
schema.addColumn("age", Type.INT8);
schema.addColumn("score", Type.FLOAT4);
TableMeta meta = CatalogUtil.newTableMeta(storeType);
Path tablePath = new Path(testDir, "testProjection.data");
Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(meta, schema, tablePath);
appender.init();
int tupleNum = 10000;
VTuple vTuple;
for(int i = 0; i < tupleNum; i++) {
vTuple = new VTuple(3);
vTuple.put(0, DatumFactory.createInt4(i + 1));
vTuple.put(1, DatumFactory.createInt8(i + 2));
vTuple.put(2, DatumFactory.createFloat4(i + 3));
appender.addTuple(vTuple);
}
appender.close();
FileStatus status = fs.getFileStatus(tablePath);
FileFragment fragment = new FileFragment("testReadAndWrite", tablePath, 0, status.getLen());
Schema target = new Schema();
target.addColumn("age", Type.INT8);
target.addColumn("score", Type.FLOAT4);
Scanner scanner = StorageManagerFactory.getStorageManager(conf).getScanner(meta, schema, fragment, target);
scanner.init();
int tupleCnt = 0;
Tuple tuple;
while ((tuple = scanner.next()) != null) {
if (storeType == StoreType.RCFILE || storeType == StoreType.TREVNI || storeType == StoreType.CSV) {
assertTrue(tuple.get(0) == null || tuple.get(0) instanceof NullDatum);
}
assertTrue(tupleCnt + 2 == tuple.get(1).asInt8());
assertTrue(tupleCnt + 3 == tuple.get(2).asFloat4());
tupleCnt++;
}
scanner.close();
assertEquals(tupleNum, tupleCnt);
}
示例10: testProjection
@Test
public void testProjection() throws IOException {
Schema schema = new Schema();
schema.addColumn("id", Type.INT4);
schema.addColumn("age", Type.INT8);
schema.addColumn("score", Type.FLOAT4);
TableMeta meta = CatalogUtil.newTableMeta(storeType);
meta.setOptions(StorageUtil.newPhysicalProperties(storeType));
if (storeType == StoreType.AVRO) {
meta.putOption(StorageConstants.AVRO_SCHEMA_LITERAL,
TEST_PROJECTION_AVRO_SCHEMA);
}
Path tablePath = new Path(testDir, "testProjection.data");
Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(meta, schema, tablePath);
appender.init();
int tupleNum = 10000;
VTuple vTuple;
for (int i = 0; i < tupleNum; i++) {
vTuple = new VTuple(3);
vTuple.put(0, DatumFactory.createInt4(i + 1));
vTuple.put(1, DatumFactory.createInt8(i + 2));
vTuple.put(2, DatumFactory.createFloat4(i + 3));
appender.addTuple(vTuple);
}
appender.close();
FileStatus status = fs.getFileStatus(tablePath);
FileFragment fragment = new FileFragment("testReadAndWrite", tablePath, 0, status.getLen());
Schema target = new Schema();
target.addColumn("age", Type.INT8);
target.addColumn("score", Type.FLOAT4);
Scanner scanner = StorageManagerFactory.getStorageManager(conf).getScanner(meta, schema, fragment, target);
scanner.init();
int tupleCnt = 0;
Tuple tuple;
while ((tuple = scanner.next()) != null) {
if (storeType == StoreType.RCFILE
|| storeType == StoreType.TREVNI
|| storeType == StoreType.CSV
|| storeType == StoreType.PARQUET
|| storeType == StoreType.SEQUENCEFILE
|| storeType == StoreType.AVRO) {
assertTrue(tuple.get(0) == null);
}
assertTrue(tupleCnt + 2 == tuple.get(1).asInt8());
assertTrue(tupleCnt + 3 == tuple.get(2).asFloat4());
tupleCnt++;
}
scanner.close();
assertEquals(tupleNum, tupleCnt);
}
示例11: testRCFileTextSerializeDeserialize
@Test
public void testRCFileTextSerializeDeserialize() throws IOException {
if(storeType != StoreType.RCFILE) return;
Schema schema = new Schema();
schema.addColumn("col1", Type.BOOLEAN);
schema.addColumn("col2", Type.BIT);
schema.addColumn("col3", Type.CHAR, 7);
schema.addColumn("col4", Type.INT2);
schema.addColumn("col5", Type.INT4);
schema.addColumn("col6", Type.INT8);
schema.addColumn("col7", Type.FLOAT4);
schema.addColumn("col8", Type.FLOAT8);
schema.addColumn("col9", Type.TEXT);
schema.addColumn("col10", Type.BLOB);
schema.addColumn("col11", Type.INET4);
schema.addColumn("col12", Type.NULL_TYPE);
schema.addColumn("col13", CatalogUtil.newDataType(Type.PROTOBUF, TajoIdProtos.QueryIdProto.class.getName()));
Options options = new Options();
TableMeta meta = CatalogUtil.newTableMeta(storeType, options);
meta.putOption(StorageConstants.CSVFILE_SERDE, TextSerializerDeserializer.class.getName());
Path tablePath = new Path(testDir, "testVariousTypes.data");
Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(meta, schema, tablePath);
appender.enableStats();
appender.init();
QueryId queryid = new QueryId("12345", 5);
ProtobufDatumFactory factory = ProtobufDatumFactory.get(TajoIdProtos.QueryIdProto.class.getName());
Tuple tuple = new VTuple(13);
tuple.put(new Datum[] {
DatumFactory.createBool(true),
DatumFactory.createBit((byte) 0x99),
DatumFactory.createChar("jinho"),
DatumFactory.createInt2((short) 17),
DatumFactory.createInt4(59),
DatumFactory.createInt8(23l),
DatumFactory.createFloat4(77.9f),
DatumFactory.createFloat8(271.9f),
DatumFactory.createText("jinho"),
DatumFactory.createBlob("hyunsik babo".getBytes()),
DatumFactory.createInet4("192.168.0.1"),
NullDatum.get(),
factory.createDatum(queryid.getProto())
});
appender.addTuple(tuple);
appender.flush();
appender.close();
FileStatus status = fs.getFileStatus(tablePath);
assertEquals(appender.getStats().getNumBytes().longValue(), status.getLen());
FileFragment fragment = new FileFragment("table", tablePath, 0, status.getLen());
Scanner scanner = StorageManagerFactory.getStorageManager(conf).getScanner(meta, schema, fragment);
scanner.init();
Tuple retrieved;
while ((retrieved=scanner.next()) != null) {
for (int i = 0; i < tuple.size(); i++) {
assertEquals(tuple.get(i), retrieved.get(i));
}
}
scanner.close();
assertEquals(appender.getStats().getNumBytes().longValue(), scanner.getInputStats().getNumBytes().longValue());
assertEquals(appender.getStats().getNumRows().longValue(), scanner.getInputStats().getNumRows().longValue());
}
示例12: testRCFileBinarySerializeDeserialize
@Test
public void testRCFileBinarySerializeDeserialize() throws IOException {
if(storeType != StoreType.RCFILE) return;
Schema schema = new Schema();
schema.addColumn("col1", Type.BOOLEAN);
schema.addColumn("col2", Type.BIT);
schema.addColumn("col3", Type.CHAR, 7);
schema.addColumn("col4", Type.INT2);
schema.addColumn("col5", Type.INT4);
schema.addColumn("col6", Type.INT8);
schema.addColumn("col7", Type.FLOAT4);
schema.addColumn("col8", Type.FLOAT8);
schema.addColumn("col9", Type.TEXT);
schema.addColumn("col10", Type.BLOB);
schema.addColumn("col11", Type.INET4);
schema.addColumn("col12", Type.NULL_TYPE);
schema.addColumn("col13", CatalogUtil.newDataType(Type.PROTOBUF, TajoIdProtos.QueryIdProto.class.getName()));
Options options = new Options();
TableMeta meta = CatalogUtil.newTableMeta(storeType, options);
meta.putOption(StorageConstants.RCFILE_SERDE, BinarySerializerDeserializer.class.getName());
Path tablePath = new Path(testDir, "testVariousTypes.data");
Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(meta, schema, tablePath);
appender.enableStats();
appender.init();
QueryId queryid = new QueryId("12345", 5);
ProtobufDatumFactory factory = ProtobufDatumFactory.get(TajoIdProtos.QueryIdProto.class.getName());
Tuple tuple = new VTuple(13);
tuple.put(new Datum[] {
DatumFactory.createBool(true),
DatumFactory.createBit((byte) 0x99),
DatumFactory.createChar("jinho"),
DatumFactory.createInt2((short) 17),
DatumFactory.createInt4(59),
DatumFactory.createInt8(23l),
DatumFactory.createFloat4(77.9f),
DatumFactory.createFloat8(271.9f),
DatumFactory.createText("jinho"),
DatumFactory.createBlob("hyunsik babo".getBytes()),
DatumFactory.createInet4("192.168.0.1"),
NullDatum.get(),
factory.createDatum(queryid.getProto())
});
appender.addTuple(tuple);
appender.flush();
appender.close();
FileStatus status = fs.getFileStatus(tablePath);
assertEquals(appender.getStats().getNumBytes().longValue(), status.getLen());
FileFragment fragment = new FileFragment("table", tablePath, 0, status.getLen());
Scanner scanner = StorageManagerFactory.getStorageManager(conf).getScanner(meta, schema, fragment);
scanner.init();
Tuple retrieved;
while ((retrieved=scanner.next()) != null) {
for (int i = 0; i < tuple.size(); i++) {
assertEquals(tuple.get(i), retrieved.get(i));
}
}
scanner.close();
assertEquals(appender.getStats().getNumBytes().longValue(), scanner.getInputStats().getNumBytes().longValue());
assertEquals(appender.getStats().getNumRows().longValue(), scanner.getInputStats().getNumRows().longValue());
}
示例13: testProjection
@Test
public void testProjection() throws IOException {
Schema schema = new Schema();
schema.addColumn("id", Type.INT4);
schema.addColumn("age", Type.INT8);
schema.addColumn("score", Type.FLOAT4);
TableMeta meta = CatalogUtil.newTableMeta(storeType);
meta.setOptions(StorageUtil.newPhysicalProperties(storeType));
if (storeType == StoreType.AVRO) {
meta.putOption(StorageConstants.AVRO_SCHEMA_LITERAL,
TEST_PROJECTION_AVRO_SCHEMA);
}
Path tablePath = new Path(testDir, "testProjection.data");
Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(meta, schema, tablePath);
appender.init();
int tupleNum = 10000;
VTuple vTuple;
for(int i = 0; i < tupleNum; i++) {
vTuple = new VTuple(3);
vTuple.put(0, DatumFactory.createInt4(i + 1));
vTuple.put(1, DatumFactory.createInt8(i + 2));
vTuple.put(2, DatumFactory.createFloat4(i + 3));
appender.addTuple(vTuple);
}
appender.close();
FileStatus status = fs.getFileStatus(tablePath);
FileFragment fragment = new FileFragment("testReadAndWrite", tablePath, 0, status.getLen());
Schema target = new Schema();
target.addColumn("age", Type.INT8);
target.addColumn("score", Type.FLOAT4);
Scanner scanner = StorageManagerFactory.getStorageManager(conf).getScanner(meta, schema, fragment, target);
scanner.init();
int tupleCnt = 0;
Tuple tuple;
while ((tuple = scanner.next()) != null) {
if (storeType == StoreType.RCFILE
|| storeType == StoreType.TREVNI
|| storeType == StoreType.CSV
|| storeType == StoreType.PARQUET
|| storeType == StoreType.AVRO) {
assertTrue(tuple.get(0) == null || tuple.get(0) instanceof NullDatum);
}
assertTrue(tupleCnt + 2 == tuple.get(1).asInt8());
assertTrue(tupleCnt + 3 == tuple.get(2).asFloat4());
tupleCnt++;
}
scanner.close();
assertEquals(tupleNum, tupleCnt);
}