当前位置: 首页>>代码示例>>Java>>正文


Java ProtobufDatumFactory类代码示例

本文整理汇总了Java中org.apache.tajo.datum.ProtobufDatumFactory的典型用法代码示例。如果您正苦于以下问题:Java ProtobufDatumFactory类的具体用法?Java ProtobufDatumFactory怎么用?Java ProtobufDatumFactory使用的例子?那么恭喜您, 这里精选的类代码示例或许可以为您提供帮助。


ProtobufDatumFactory类属于org.apache.tajo.datum包,在下文中一共展示了ProtobufDatumFactory类的15个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Java代码示例。

示例1: testCreate

import org.apache.tajo.datum.ProtobufDatumFactory; //导入依赖的package包/类
@Test
public void testCreate() throws IOException, NoSuchMethodException, ClassNotFoundException {
  TajoDataTypes.DataType.Builder builder = TajoDataTypes.DataType.newBuilder();
  builder.setType(TajoDataTypes.Type.PROTOBUF);
  builder.setCode(TajoIdProtos.QueryIdProto.class.getName());

  ProtobufDatumFactory factory = ProtobufDatumFactory.get(builder.build());
  TajoIdProtos.QueryIdProto.Builder queryIdBuilder = factory.newBuilder();
  queryIdBuilder.setId(String.valueOf(System.currentTimeMillis()));
  queryIdBuilder.setSeq(1);

  TajoIdProtos.QueryIdProto queryId = queryIdBuilder.build();
  ProtobufDatum datum = factory.createDatum(queryId);

  ProtobufJsonFormat formatter = ProtobufJsonFormat.getInstance();
  String json = formatter.printToString(datum.get());

  TajoIdProtos.QueryIdProto.Builder fromJson = factory.newBuilder();
  formatter.merge(TextUtils.toInputStream(json), fromJson);
  assertEquals(queryId, fromJson.build());
}
 
开发者ID:apache,项目名称:tajo,代码行数:22,代码来源:TestProtobufDatum.java

示例2: addBinary

import org.apache.tajo.datum.ProtobufDatumFactory; //导入依赖的package包/类
@Override
final public void addBinary(Binary value) {
  try {
    ProtobufDatumFactory factory =
        ProtobufDatumFactory.get(dataType.getCode());
    Message.Builder builder = factory.newBuilder();
    builder.mergeFrom(value.getBytes());
    parent.add(factory.createDatum(builder));
  } catch (InvalidProtocolBufferException e) {
    throw new RuntimeException(e);
  }
}
 
开发者ID:gruter,项目名称:tajo-cdh,代码行数:13,代码来源:TajoRecordConverter.java

示例3: testVariousTypes

import org.apache.tajo.datum.ProtobufDatumFactory; //导入依赖的package包/类
@Test
public void testVariousTypes() throws IOException {
  SchemaBuilder schemaBld = SchemaBuilder.builder()
      .add("col1", Type.BOOLEAN)
      .add("col2", CatalogUtil.newDataTypeWithLen(Type.CHAR, 7))
      .add("col3", Type.INT2)
      .add("col4", Type.INT4)
      .add("col5", Type.INT8)
      .add("col6", Type.FLOAT4)
      .add("col7", Type.FLOAT8)
      .add("col8", Type.TEXT)
      .add("col9", Type.BLOB);
  if (dateTypeSupport()) {
    schemaBld.add("col10", Type.DATE);
  }
  if (protoTypeSupport()) {
    schemaBld.add("col11", CatalogUtil.newDataType(Type.PROTOBUF, TajoIdProtos.QueryIdProto.class.getName()));
  }

  Schema schema = schemaBld.build();

  TableMeta meta = CatalogUtil.newTableMeta(dataFormat, conf);
  if (dataFormat.equalsIgnoreCase(BuiltinStorages.AVRO)) {
    String path = JavaResourceUtil.getResourceURL("dataset/testVariousTypes.avsc").toString();
    meta.putProperty(StorageConstants.AVRO_SCHEMA_URL, path);
  }

  FileTablespace sm = TablespaceManager.getLocalFs();
  Path tablePath = new Path(testDir, "testVariousTypes.data");
  Appender appender = sm.getAppender(meta, schema, tablePath);
  appender.init();

  QueryId queryid = new QueryId("12345", 5);

  VTuple tuple = new VTuple(9 + (dateTypeSupport() ? 1 : 0) + (protoTypeSupport() ? 1 : 0));
  tuple.put(new Datum[] {
      DatumFactory.createBool(true),
      DatumFactory.createChar("hyunsik"),
      DatumFactory.createInt2((short) 17),
      DatumFactory.createInt4(59),
      DatumFactory.createInt8(23L),
      DatumFactory.createFloat4(77.9f),
      DatumFactory.createFloat8(271.9f),
      DatumFactory.createText("hyunsik"),
      DatumFactory.createBlob("hyunsik".getBytes()),
  });

  short currentIdx = 9;

  if (dateTypeSupport()) {
    tuple.put(currentIdx, DatumFactory.createDate(2016, 6, 28));
    currentIdx++;
  }

  if (protoTypeSupport()) {
    tuple.put(currentIdx, ProtobufDatumFactory.createDatum(queryid.getProto()));
  }

  appender.addTuple(tuple);
  appender.flush();
  appender.close();

  FileStatus status = fs.getFileStatus(tablePath);
  FileFragment fragment = new FileFragment("table", tablePath, 0, status.getLen());
  Scanner scanner =  sm.getScanner(meta, schema, fragment, null);
  scanner.init();

  Tuple retrieved;
  while ((retrieved = scanner.next()) != null) {
    for (int i = 0; i < tuple.size(); i++) {
      assertEquals(tuple.get(i), retrieved.asDatum(i));
    }
  }
  scanner.close();
}
 
开发者ID:apache,项目名称:tajo,代码行数:76,代码来源:TestStorages.java

示例4: testRCFileTextSerializeDeserialize

import org.apache.tajo.datum.ProtobufDatumFactory; //导入依赖的package包/类
@Test
public void testRCFileTextSerializeDeserialize() throws IOException {
  if(!dataFormat.equalsIgnoreCase(BuiltinStorages.RCFILE)) return;

  Schema schema = SchemaBuilder.builder()
      .add("col1", Type.BOOLEAN)
      .add("col2", CatalogUtil.newDataTypeWithLen(Type.CHAR, 7))
      .add("col3", Type.INT2)
      .add("col4", Type.INT4)
      .add("col5", Type.INT8)
      .add("col6", Type.FLOAT4)
      .add("col7", Type.FLOAT8)
      .add("col8", Type.TEXT)
      .add("col9", Type.BLOB)
      .add("col10", CatalogUtil.newDataType(Type.PROTOBUF, TajoIdProtos.QueryIdProto.class.getName()))
      .build();

  TableMeta meta = CatalogUtil.newTableMeta(dataFormat, conf);
  meta.putProperty(StorageConstants.CSVFILE_SERDE, TextSerializerDeserializer.class.getName());

  Path tablePath = new Path(testDir, "testRCFileTextSerializeDeserialize.data");
  FileTablespace sm = TablespaceManager.getLocalFs();
  Appender appender = sm.getAppender(meta, schema, tablePath);
  appender.enableStats();
  appender.init();

  QueryId queryid = new QueryId("12345", 5);
  ProtobufDatumFactory factory = ProtobufDatumFactory.get(TajoIdProtos.QueryIdProto.class.getName());

  VTuple tuple = new VTuple(new Datum[] {
      DatumFactory.createBool(true),
      DatumFactory.createChar("jinho"),
      DatumFactory.createInt2((short) 17),
      DatumFactory.createInt4(59),
      DatumFactory.createInt8(23l),
      DatumFactory.createFloat4(77.9f),
      DatumFactory.createFloat8(271.9f),
      DatumFactory.createText("jinho"),
      DatumFactory.createBlob("hyunsik babo".getBytes()),
      factory.createDatum(queryid.getProto())
  });
  appender.addTuple(tuple);
  appender.flush();
  appender.close();

  FileStatus status = fs.getFileStatus(tablePath);
  assertEquals(appender.getStats().getNumBytes().longValue(), status.getLen());

  FileFragment fragment = new FileFragment("table", tablePath, 0, status.getLen());
  Scanner scanner =  TablespaceManager.getLocalFs().getScanner(meta, schema, fragment, null);
  scanner.init();

  Tuple retrieved;
  while ((retrieved=scanner.next()) != null) {
    for (int i = 0; i < tuple.size(); i++) {
      assertEquals(tuple.get(i), retrieved.asDatum(i));
    }
  }
  scanner.close();
  assertEquals(appender.getStats().getNumBytes().longValue(), scanner.getInputStats().getNumBytes().longValue());
  assertEquals(appender.getStats().getNumRows().longValue(), scanner.getInputStats().getNumRows().longValue());
}
 
开发者ID:apache,项目名称:tajo,代码行数:63,代码来源:TestStorages.java

示例5: testRCFileBinarySerializeDeserialize

import org.apache.tajo.datum.ProtobufDatumFactory; //导入依赖的package包/类
@Test
public void testRCFileBinarySerializeDeserialize() throws IOException {
  if(!dataFormat.equalsIgnoreCase(BuiltinStorages.RCFILE)) return;

  Schema schema = SchemaBuilder.builder()
      .add("col1", Type.BOOLEAN)
      .add("col2", CatalogUtil.newDataTypeWithLen(Type.CHAR, 7))
      .add("col3", Type.INT2)
      .add("col4", Type.INT4)
      .add("col5", Type.INT8)
      .add("col6", Type.FLOAT4)
      .add("col7", Type.FLOAT8)
      .add("col8", Type.TEXT)
      .add("col9", Type.BLOB)
      .add("col10", CatalogUtil.newDataType(Type.PROTOBUF, TajoIdProtos.QueryIdProto.class.getName()))
      .build();

  TableMeta meta = CatalogUtil.newTableMeta(dataFormat, conf);
  meta.putProperty(StorageConstants.RCFILE_SERDE, BinarySerializerDeserializer.class.getName());

  Path tablePath = new Path(testDir, "testRCFileBinarySerializeDeserialize.data");
  FileTablespace sm = TablespaceManager.getLocalFs();
  Appender appender = sm.getAppender(meta, schema, tablePath);
  appender.enableStats();
  appender.init();

  QueryId queryid = new QueryId("12345", 5);
  ProtobufDatumFactory factory = ProtobufDatumFactory.get(TajoIdProtos.QueryIdProto.class.getName());

  VTuple tuple = new VTuple(new Datum[] {
      DatumFactory.createBool(true),
      DatumFactory.createChar("jinho"),
      DatumFactory.createInt2((short) 17),
      DatumFactory.createInt4(59),
      DatumFactory.createInt8(23l),
      DatumFactory.createFloat4(77.9f),
      DatumFactory.createFloat8(271.9f),
      DatumFactory.createText("jinho"),
      DatumFactory.createBlob("hyunsik babo".getBytes()),
      factory.createDatum(queryid.getProto())
  });
  appender.addTuple(tuple);
  appender.flush();
  appender.close();

  FileStatus status = fs.getFileStatus(tablePath);
  assertEquals(appender.getStats().getNumBytes().longValue(), status.getLen());

  FileFragment fragment = new FileFragment("table", tablePath, 0, status.getLen());
  Scanner scanner =  TablespaceManager.getLocalFs().getScanner(meta, schema, fragment, null);
  scanner.init();

  Tuple retrieved;
  while ((retrieved=scanner.next()) != null) {
    for (int i = 0; i < tuple.size(); i++) {
      assertEquals(tuple.get(i), retrieved.asDatum(i));
    }
  }
  scanner.close();
  assertEquals(appender.getStats().getNumBytes().longValue(), scanner.getInputStats().getNumBytes().longValue());
  assertEquals(appender.getStats().getNumRows().longValue(), scanner.getInputStats().getNumRows().longValue());
}
 
开发者ID:apache,项目名称:tajo,代码行数:63,代码来源:TestStorages.java

示例6: testSequenceFileTextSerializeDeserialize

import org.apache.tajo.datum.ProtobufDatumFactory; //导入依赖的package包/类
@Test
public void testSequenceFileTextSerializeDeserialize() throws IOException {
  if(!dataFormat.equalsIgnoreCase(BuiltinStorages.SEQUENCE_FILE)) return;

  Schema schema = SchemaBuilder.builder()
      .add("col1", Type.BOOLEAN)
      .add("col2", CatalogUtil.newDataTypeWithLen(Type.CHAR, 7))
      .add("col3", Type.INT2)
      .add("col4", Type.INT4)
      .add("col5", Type.INT8)
      .add("col6", Type.FLOAT4)
      .add("col7", Type.FLOAT8)
      .add("col8", Type.TEXT)
      .add("col9", Type.BLOB)
      .add("col10", CatalogUtil.newDataType(Type.PROTOBUF, TajoIdProtos.QueryIdProto.class.getName())).build();

  TableMeta meta = CatalogUtil.newTableMeta(dataFormat, conf);
  meta.putProperty(StorageConstants.SEQUENCEFILE_SERDE, TextSerializerDeserializer.class.getName());

  Path tablePath = new Path(testDir, "testSequenceFileTextSerializeDeserialize.data");
  FileTablespace sm = TablespaceManager.getLocalFs();
  Appender appender = sm.getAppender(meta, schema, tablePath);
  appender.enableStats();
  appender.init();

  QueryId queryid = new QueryId("12345", 5);

  VTuple tuple = new VTuple(new Datum[] {
      DatumFactory.createBool(true),
      DatumFactory.createChar("jinho"),
      DatumFactory.createInt2((short) 17),
      DatumFactory.createInt4(59),
      DatumFactory.createInt8(23l),
      DatumFactory.createFloat4(77.9f),
      DatumFactory.createFloat8(271.9f),
      DatumFactory.createText("jinho"),
      DatumFactory.createBlob("hyunsik babo".getBytes()),
      ProtobufDatumFactory.createDatum(queryid.getProto())
  });
  appender.addTuple(tuple);
  appender.flush();
  appender.close();

  FileStatus status = fs.getFileStatus(tablePath);
  assertEquals(appender.getStats().getNumBytes().longValue(), status.getLen());

  FileFragment fragment = new FileFragment("table", tablePath, 0, status.getLen());
  Scanner scanner =  TablespaceManager.getLocalFs().getScanner(meta, schema, fragment, null);
  scanner.init();

  assertTrue(scanner instanceof SequenceFileScanner);
  Writable key = ((SequenceFileScanner) scanner).getKey();
  assertEquals(key.getClass().getCanonicalName(), LongWritable.class.getCanonicalName());

  Tuple retrieved;
  while ((retrieved=scanner.next()) != null) {
    for (int i = 0; i < tuple.size(); i++) {
      assertEquals(tuple.get(i), retrieved.asDatum(i));
    }
  }
  scanner.close();
  assertEquals(appender.getStats().getNumBytes().longValue(), scanner.getInputStats().getNumBytes().longValue());
  assertEquals(appender.getStats().getNumRows().longValue(), scanner.getInputStats().getNumRows().longValue());
}
 
开发者ID:apache,项目名称:tajo,代码行数:65,代码来源:TestStorages.java

示例7: testSequenceFileTextSerializeDeserializeWithDeprecatedProperties

import org.apache.tajo.datum.ProtobufDatumFactory; //导入依赖的package包/类
@Test
public void testSequenceFileTextSerializeDeserializeWithDeprecatedProperties() throws IOException {
  if(!dataFormat.equalsIgnoreCase(BuiltinStorages.SEQUENCE_FILE)) return;

  Schema schema = SchemaBuilder.builder()
    .add("col1", Type.BOOLEAN)
    .add("col2", CatalogUtil.newDataTypeWithLen(Type.CHAR, 7))
    .add("col3", Type.INT2)
    .add("col4", Type.INT4)
    .add("col5", Type.INT8)
    .add("col6", Type.FLOAT4)
    .add("col7", Type.FLOAT8)
    .add("col8", Type.TEXT)
    .add("col9", Type.BLOB)
    .add("col10", CatalogUtil.newDataType(Type.PROTOBUF, TajoIdProtos.QueryIdProto.class.getName())).build();

  TableMeta meta = CatalogUtil.newTableMeta(dataFormat, conf);
  meta.putProperty(StorageConstants.SEQUENCEFILE_SERDE, TextSerializerDeserializer.class.getName());
  meta.putProperty(StorageConstants.SEQUENCEFILE_DELIMITER, "\u0001");
  meta.putProperty(StorageConstants.SEQUENCEFILE_NULL, "\\");

  Path tablePath = new Path(testDir, "testSequenceFileTextSerializeDeserializeWithDeprecatedProperties.data");
  FileTablespace sm = TablespaceManager.getLocalFs();
  Appender appender = sm.getAppender(meta, schema, tablePath);
  appender.enableStats();
  appender.init();

  QueryId queryid = new QueryId("12345", 5);

  VTuple tuple = new VTuple(new Datum[] {
    DatumFactory.createBool(true),
    DatumFactory.createChar("jinho"),
    DatumFactory.createInt2((short) 17),
    DatumFactory.createInt4(59),
    DatumFactory.createInt8(23l),
    DatumFactory.createFloat4(77.9f),
    DatumFactory.createFloat8(271.9f),
    DatumFactory.createText("jinho"),
    DatumFactory.createBlob("hyunsik babo".getBytes()),
    ProtobufDatumFactory.createDatum(queryid.getProto())
  });
  appender.addTuple(tuple);
  appender.flush();
  appender.close();

  FileStatus status = fs.getFileStatus(tablePath);
  assertEquals(appender.getStats().getNumBytes().longValue(), status.getLen());

  FileFragment fragment = new FileFragment("table", tablePath, 0, status.getLen());
  Scanner scanner =  TablespaceManager.getLocalFs().getScanner(meta, schema, fragment, null);
  scanner.init();

  assertTrue(scanner instanceof SequenceFileScanner);
  Writable key = ((SequenceFileScanner) scanner).getKey();
  assertEquals(key.getClass().getCanonicalName(), LongWritable.class.getCanonicalName());

  Tuple retrieved;
  while ((retrieved=scanner.next()) != null) {
    for (int i = 0; i < tuple.size(); i++) {
      assertEquals(tuple.get(i), retrieved.asDatum(i));
    }
  }
  scanner.close();
  assertEquals(appender.getStats().getNumBytes().longValue(), scanner.getInputStats().getNumBytes().longValue());
  assertEquals(appender.getStats().getNumRows().longValue(), scanner.getInputStats().getNumRows().longValue());
}
 
开发者ID:apache,项目名称:tajo,代码行数:67,代码来源:TestStorages.java

示例8: testSequenceFileTextSerializeDeserializeWithNonDeprecatedProperties

import org.apache.tajo.datum.ProtobufDatumFactory; //导入依赖的package包/类
@Test
public void testSequenceFileTextSerializeDeserializeWithNonDeprecatedProperties() throws IOException {
  if(!dataFormat.equalsIgnoreCase(BuiltinStorages.SEQUENCE_FILE)) return;

  Schema schema = SchemaBuilder.builder()
    .add("col1", Type.BOOLEAN)
    .add("col2", CatalogUtil.newDataTypeWithLen(Type.CHAR, 7))
    .add("col3", Type.INT2)
    .add("col4", Type.INT4)
    .add("col5", Type.INT8)
    .add("col6", Type.FLOAT4)
    .add("col7", Type.FLOAT8)
    .add("col8", Type.TEXT)
    .add("col9", Type.BLOB)
    .add("col10", CatalogUtil.newDataType(Type.PROTOBUF, TajoIdProtos.QueryIdProto.class.getName())).build();

  TableMeta meta = CatalogUtil.newTableMeta(dataFormat, conf);
  meta.putProperty(StorageConstants.SEQUENCEFILE_SERDE, TextSerializerDeserializer.class.getName());
  meta.putProperty(StorageConstants.TEXT_DELIMITER, "\u0001");
  meta.putProperty(StorageConstants.TEXT_NULL, "\\");

  Path tablePath = new Path(testDir, "testSequenceFileTextSerializeDeserializeWithNonDeprecatedProperties.data");
  FileTablespace sm = TablespaceManager.getLocalFs();
  Appender appender = sm.getAppender(meta, schema, tablePath);
  appender.enableStats();
  appender.init();

  QueryId queryid = new QueryId("12345", 5);

  VTuple tuple = new VTuple(new Datum[] {
    DatumFactory.createBool(true),
    DatumFactory.createChar("jinho"),
    DatumFactory.createInt2((short) 17),
    DatumFactory.createInt4(59),
    DatumFactory.createInt8(23l),
    DatumFactory.createFloat4(77.9f),
    DatumFactory.createFloat8(271.9f),
    DatumFactory.createText("jinho"),
    DatumFactory.createBlob("hyunsik babo".getBytes()),
    ProtobufDatumFactory.createDatum(queryid.getProto())
  });
  appender.addTuple(tuple);
  appender.flush();
  appender.close();

  FileStatus status = fs.getFileStatus(tablePath);
  assertEquals(appender.getStats().getNumBytes().longValue(), status.getLen());

  FileFragment fragment = new FileFragment("table", tablePath, 0, status.getLen());
  Scanner scanner =  TablespaceManager.getLocalFs().getScanner(meta, schema, fragment, null);
  scanner.init();

  assertTrue(scanner instanceof SequenceFileScanner);
  Writable key = ((SequenceFileScanner) scanner).getKey();
  assertEquals(key.getClass().getCanonicalName(), LongWritable.class.getCanonicalName());

  Tuple retrieved;
  while ((retrieved=scanner.next()) != null) {
    for (int i = 0; i < tuple.size(); i++) {
      assertEquals(tuple.get(i), retrieved.asDatum(i));
    }
  }
  scanner.close();
  assertEquals(appender.getStats().getNumBytes().longValue(), scanner.getInputStats().getNumBytes().longValue());
  assertEquals(appender.getStats().getNumRows().longValue(), scanner.getInputStats().getNumRows().longValue());
}
 
开发者ID:apache,项目名称:tajo,代码行数:67,代码来源:TestStorages.java

示例9: testSequenceFileBinarySerializeDeserialize

import org.apache.tajo.datum.ProtobufDatumFactory; //导入依赖的package包/类
@Test
public void testSequenceFileBinarySerializeDeserialize() throws IOException {
  if(!dataFormat.equalsIgnoreCase(BuiltinStorages.SEQUENCE_FILE)) return;

  Schema schema = SchemaBuilder.builder()
      .add("col1", Type.BOOLEAN)
      .add("col2", CatalogUtil.newDataTypeWithLen(Type.CHAR, 7))
      .add("col3", Type.INT2)
      .add("col4", Type.INT4)
      .add("col5", Type.INT8)
      .add("col6", Type.FLOAT4)
      .add("col7", Type.FLOAT8)
      .add("col8", Type.TEXT)
      .add("col9", Type.BLOB)
      .add("col10", CatalogUtil.newDataType(Type.PROTOBUF, TajoIdProtos.QueryIdProto.class.getName()))
      .build();

  TableMeta meta = CatalogUtil.newTableMeta(dataFormat, conf);
  meta.putProperty(StorageConstants.SEQUENCEFILE_SERDE, BinarySerializerDeserializer.class.getName());

  Path tablePath = new Path(testDir, "testVariousTypes.data");
  FileTablespace sm = TablespaceManager.getLocalFs();
  Appender appender = sm.getAppender(meta, schema, tablePath);
  appender.enableStats();
  appender.init();

  QueryId queryid = new QueryId("12345", 5);

  VTuple tuple = new VTuple(new Datum[] {
      DatumFactory.createBool(true),
      DatumFactory.createChar("jinho"),
      DatumFactory.createInt2((short) 17),
      DatumFactory.createInt4(59),
      DatumFactory.createInt8(23l),
      DatumFactory.createFloat4(77.9f),
      DatumFactory.createFloat8(271.9f),
      DatumFactory.createText("jinho"),
      DatumFactory.createBlob("hyunsik babo".getBytes()),
      ProtobufDatumFactory.createDatum(queryid.getProto())
  });
  appender.addTuple(tuple);
  appender.flush();
  appender.close();

  FileStatus status = fs.getFileStatus(tablePath);
  assertEquals(appender.getStats().getNumBytes().longValue(), status.getLen());

  FileFragment fragment = new FileFragment("table", tablePath, 0, status.getLen());
  Scanner scanner = TablespaceManager.getLocalFs().getScanner(meta, schema, fragment, null);
  scanner.init();

  assertTrue(scanner instanceof SequenceFileScanner);
  Writable key = ((SequenceFileScanner) scanner).getKey();
  assertEquals(key.getClass().getCanonicalName(), BytesWritable.class.getCanonicalName());

  Tuple retrieved;
  while ((retrieved=scanner.next()) != null) {
    for (int i = 0; i < tuple.size(); i++) {
      assertEquals(tuple.get(i), retrieved.asDatum(i));
    }
  }
  scanner.close();
  assertEquals(appender.getStats().getNumBytes().longValue(), scanner.getInputStats().getNumBytes().longValue());
  assertEquals(appender.getStats().getNumRows().longValue(), scanner.getInputStats().getNumRows().longValue());
}
 
开发者ID:apache,项目名称:tajo,代码行数:66,代码来源:TestStorages.java

示例10: testVariousTypes

import org.apache.tajo.datum.ProtobufDatumFactory; //导入依赖的package包/类
@Test
public void testVariousTypes() throws IOException {
  Schema schema = new Schema();
  schema.addColumn("col1", Type.BOOLEAN);
  schema.addColumn("col2", Type.BIT);
  schema.addColumn("col3", Type.CHAR, 7);
  schema.addColumn("col4", Type.INT2);
  schema.addColumn("col5", Type.INT4);
  schema.addColumn("col6", Type.INT8);
  schema.addColumn("col7", Type.FLOAT4);
  schema.addColumn("col8", Type.FLOAT8);
  schema.addColumn("col9", Type.TEXT);
  schema.addColumn("col10", Type.BLOB);
  schema.addColumn("col11", Type.INET4);
  schema.addColumn("col12", Type.NULL_TYPE);
  schema.addColumn("col13", CatalogUtil.newDataType(Type.PROTOBUF, TajoIdProtos.QueryIdProto.class.getName()));

  Options options = new Options();
  TableMeta meta = CatalogUtil.newTableMeta(storeType, options);

  Path tablePath = new Path(testDir, "testVariousTypes.data");
  Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(meta, schema, tablePath);
  appender.init();

  QueryId queryid = new QueryId("12345", 5);
  ProtobufDatumFactory factory = ProtobufDatumFactory.get(TajoIdProtos.QueryIdProto.class.getName());

  Tuple tuple = new VTuple(13);
  tuple.put(new Datum[] {
      DatumFactory.createBool(true),
      DatumFactory.createBit((byte) 0x99),
      DatumFactory.createChar("hyunsik"),
      DatumFactory.createInt2((short) 17),
      DatumFactory.createInt4(59),
      DatumFactory.createInt8(23l),
      DatumFactory.createFloat4(77.9f),
      DatumFactory.createFloat8(271.9f),
      DatumFactory.createText("hyunsik"),
      DatumFactory.createBlob("hyunsik".getBytes()),
      DatumFactory.createInet4("192.168.0.1"),
      NullDatum.get(),
      factory.createDatum(queryid.getProto())
  });
  appender.addTuple(tuple);
  appender.flush();
  appender.close();

  FileStatus status = fs.getFileStatus(tablePath);
  FileFragment fragment = new FileFragment("table", tablePath, 0, status.getLen());
  Scanner scanner =  StorageManagerFactory.getStorageManager(conf).getScanner(meta, schema, fragment);
  scanner.init();

  Tuple retrieved;
  while ((retrieved=scanner.next()) != null) {
    for (int i = 0; i < tuple.size(); i++) {
      assertEquals(tuple.get(i), retrieved.get(i));
    }
  }
  scanner.close();
}
 
开发者ID:apache,项目名称:incubator-tajo,代码行数:61,代码来源:TestStorages.java

示例11: testRCFileTextSerializeDeserialize

import org.apache.tajo.datum.ProtobufDatumFactory; //导入依赖的package包/类
@Test
public void testRCFileTextSerializeDeserialize() throws IOException {
  if(storeType != StoreType.RCFILE) return;

  Schema schema = new Schema();
  schema.addColumn("col1", Type.BOOLEAN);
  schema.addColumn("col2", Type.BIT);
  schema.addColumn("col3", Type.CHAR, 7);
  schema.addColumn("col4", Type.INT2);
  schema.addColumn("col5", Type.INT4);
  schema.addColumn("col6", Type.INT8);
  schema.addColumn("col7", Type.FLOAT4);
  schema.addColumn("col8", Type.FLOAT8);
  schema.addColumn("col9", Type.TEXT);
  schema.addColumn("col10", Type.BLOB);
  schema.addColumn("col11", Type.INET4);
  schema.addColumn("col12", Type.NULL_TYPE);
  schema.addColumn("col13", CatalogUtil.newDataType(Type.PROTOBUF, TajoIdProtos.QueryIdProto.class.getName()));

  Options options = new Options();
  TableMeta meta = CatalogUtil.newTableMeta(storeType, options);
  meta.putOption(CatalogConstants.CSVFILE_SERDE, TextSerializerDeserializer.class.getName());

  Path tablePath = new Path(testDir, "testVariousTypes.data");
  Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(meta, schema, tablePath);
  appender.enableStats();
  appender.init();

  QueryId queryid = new QueryId("12345", 5);
  ProtobufDatumFactory factory = ProtobufDatumFactory.get(TajoIdProtos.QueryIdProto.class.getName());

  Tuple tuple = new VTuple(13);
  tuple.put(new Datum[] {
      DatumFactory.createBool(true),
      DatumFactory.createBit((byte) 0x99),
      DatumFactory.createChar("jinho"),
      DatumFactory.createInt2((short) 17),
      DatumFactory.createInt4(59),
      DatumFactory.createInt8(23l),
      DatumFactory.createFloat4(77.9f),
      DatumFactory.createFloat8(271.9f),
      DatumFactory.createText("jinho"),
      DatumFactory.createBlob("hyunsik babo".getBytes()),
      DatumFactory.createInet4("192.168.0.1"),
      NullDatum.get(),
      factory.createDatum(queryid.getProto())
  });
  appender.addTuple(tuple);
  appender.flush();
  appender.close();

  FileStatus status = fs.getFileStatus(tablePath);
  assertEquals(appender.getStats().getNumBytes().longValue(), status.getLen());

  FileFragment fragment = new FileFragment("table", tablePath, 0, status.getLen());
  Scanner scanner =  StorageManagerFactory.getStorageManager(conf).getScanner(meta, schema, fragment);
  scanner.init();

  Tuple retrieved;
  while ((retrieved=scanner.next()) != null) {
    for (int i = 0; i < tuple.size(); i++) {
      assertEquals(tuple.get(i), retrieved.get(i));
    }
  }
  scanner.close();
  assertEquals(appender.getStats().getNumBytes().longValue(), scanner.getInputStats().getNumBytes().longValue());
  assertEquals(appender.getStats().getNumRows().longValue(), scanner.getInputStats().getNumRows().longValue());
}
 
开发者ID:apache,项目名称:incubator-tajo,代码行数:69,代码来源:TestStorages.java

示例12: testRCFileBinarySerializeDeserialize

import org.apache.tajo.datum.ProtobufDatumFactory; //导入依赖的package包/类
@Test
public void testRCFileBinarySerializeDeserialize() throws IOException {
  if(storeType != StoreType.RCFILE) return;

  Schema schema = new Schema();
  schema.addColumn("col1", Type.BOOLEAN);
  schema.addColumn("col2", Type.BIT);
  schema.addColumn("col3", Type.CHAR, 7);
  schema.addColumn("col4", Type.INT2);
  schema.addColumn("col5", Type.INT4);
  schema.addColumn("col6", Type.INT8);
  schema.addColumn("col7", Type.FLOAT4);
  schema.addColumn("col8", Type.FLOAT8);
  schema.addColumn("col9", Type.TEXT);
  schema.addColumn("col10", Type.BLOB);
  schema.addColumn("col11", Type.INET4);
  schema.addColumn("col12", Type.NULL_TYPE);
  schema.addColumn("col13", CatalogUtil.newDataType(Type.PROTOBUF, TajoIdProtos.QueryIdProto.class.getName()));

  Options options = new Options();
  TableMeta meta = CatalogUtil.newTableMeta(storeType, options);
  meta.putOption(CatalogConstants.RCFILE_SERDE, BinarySerializerDeserializer.class.getName());

  Path tablePath = new Path(testDir, "testVariousTypes.data");
  Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(meta, schema, tablePath);
  appender.enableStats();
  appender.init();

  QueryId queryid = new QueryId("12345", 5);
  ProtobufDatumFactory factory = ProtobufDatumFactory.get(TajoIdProtos.QueryIdProto.class.getName());

  Tuple tuple = new VTuple(13);
  tuple.put(new Datum[] {
      DatumFactory.createBool(true),
      DatumFactory.createBit((byte) 0x99),
      DatumFactory.createChar("jinho"),
      DatumFactory.createInt2((short) 17),
      DatumFactory.createInt4(59),
      DatumFactory.createInt8(23l),
      DatumFactory.createFloat4(77.9f),
      DatumFactory.createFloat8(271.9f),
      DatumFactory.createText("jinho"),
      DatumFactory.createBlob("hyunsik babo".getBytes()),
      DatumFactory.createInet4("192.168.0.1"),
      NullDatum.get(),
      factory.createDatum(queryid.getProto())
  });
  appender.addTuple(tuple);
  appender.flush();
  appender.close();

  FileStatus status = fs.getFileStatus(tablePath);
  assertEquals(appender.getStats().getNumBytes().longValue(), status.getLen());

  FileFragment fragment = new FileFragment("table", tablePath, 0, status.getLen());
  Scanner scanner =  StorageManagerFactory.getStorageManager(conf).getScanner(meta, schema, fragment);
  scanner.init();

  Tuple retrieved;
  while ((retrieved=scanner.next()) != null) {
    for (int i = 0; i < tuple.size(); i++) {
      assertEquals(tuple.get(i), retrieved.get(i));
    }
  }
  scanner.close();
  assertEquals(appender.getStats().getNumBytes().longValue(), scanner.getInputStats().getNumBytes().longValue());
  assertEquals(appender.getStats().getNumRows().longValue(), scanner.getInputStats().getNumRows().longValue());
}
 
开发者ID:apache,项目名称:incubator-tajo,代码行数:69,代码来源:TestStorages.java

示例13: testVariousTypes

import org.apache.tajo.datum.ProtobufDatumFactory; //导入依赖的package包/类
@Test
public void testVariousTypes() throws IOException {
  Schema schema = new Schema();
  schema.addColumn("col1", Type.BOOLEAN);
  schema.addColumn("col2", Type.BIT);
  schema.addColumn("col3", Type.CHAR, 7);
  schema.addColumn("col4", Type.INT2);
  schema.addColumn("col5", Type.INT4);
  schema.addColumn("col6", Type.INT8);
  schema.addColumn("col7", Type.FLOAT4);
  schema.addColumn("col8", Type.FLOAT8);
  schema.addColumn("col9", Type.TEXT);
  schema.addColumn("col10", Type.BLOB);
  schema.addColumn("col11", Type.INET4);
  schema.addColumn("col12", Type.NULL_TYPE);
  schema.addColumn("col13", CatalogUtil.newDataType(Type.PROTOBUF, TajoIdProtos.QueryIdProto.class.getName()));

  Options options = new Options();
  TableMeta meta = CatalogUtil.newTableMeta(storeType, options);
  meta.setOptions(StorageUtil.newPhysicalProperties(storeType));
  if (storeType == StoreType.AVRO) {
    String path = FileUtil.getResourcePath("testVariousTypes.avsc").toString();
    meta.putOption(StorageConstants.AVRO_SCHEMA_URL, path);
  }

  Path tablePath = new Path(testDir, "testVariousTypes.data");
  Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(meta, schema, tablePath);
  appender.init();

  QueryId queryid = new QueryId("12345", 5);
  ProtobufDatumFactory factory = ProtobufDatumFactory.get(TajoIdProtos.QueryIdProto.class.getName());

  Tuple tuple = new VTuple(13);
  tuple.put(new Datum[] {
      DatumFactory.createBool(true),
      DatumFactory.createBit((byte) 0x99),
      DatumFactory.createChar("hyunsik"),
      DatumFactory.createInt2((short) 17),
      DatumFactory.createInt4(59),
      DatumFactory.createInt8(23l),
      DatumFactory.createFloat4(77.9f),
      DatumFactory.createFloat8(271.9f),
      DatumFactory.createText("hyunsik"),
      DatumFactory.createBlob("hyunsik".getBytes()),
      DatumFactory.createInet4("192.168.0.1"),
      NullDatum.get(),
      factory.createDatum(queryid.getProto())
  });
  appender.addTuple(tuple);
  appender.flush();
  appender.close();

  FileStatus status = fs.getFileStatus(tablePath);
  FileFragment fragment = new FileFragment("table", tablePath, 0, status.getLen());
  Scanner scanner =  StorageManagerFactory.getStorageManager(conf).getScanner(meta, schema, fragment);
  scanner.init();

  Tuple retrieved;
  while ((retrieved = scanner.next()) != null) {
    for (int i = 0; i < tuple.size(); i++) {
      assertEquals(tuple.get(i), retrieved.get(i));
    }
  }
  scanner.close();
}
 
开发者ID:gruter,项目名称:tajo-cdh,代码行数:66,代码来源:TestStorages.java

示例14: testRCFileTextSerializeDeserialize

import org.apache.tajo.datum.ProtobufDatumFactory; //导入依赖的package包/类
@Test
public void testRCFileTextSerializeDeserialize() throws IOException {
  if(storeType != StoreType.RCFILE) return;

  Schema schema = new Schema();
  schema.addColumn("col1", Type.BOOLEAN);
  schema.addColumn("col2", Type.BIT);
  schema.addColumn("col3", Type.CHAR, 7);
  schema.addColumn("col4", Type.INT2);
  schema.addColumn("col5", Type.INT4);
  schema.addColumn("col6", Type.INT8);
  schema.addColumn("col7", Type.FLOAT4);
  schema.addColumn("col8", Type.FLOAT8);
  schema.addColumn("col9", Type.TEXT);
  schema.addColumn("col10", Type.BLOB);
  schema.addColumn("col11", Type.INET4);
  schema.addColumn("col12", Type.NULL_TYPE);
  schema.addColumn("col13", CatalogUtil.newDataType(Type.PROTOBUF, TajoIdProtos.QueryIdProto.class.getName()));

  Options options = new Options();
  TableMeta meta = CatalogUtil.newTableMeta(storeType, options);
  meta.putOption(StorageConstants.CSVFILE_SERDE, TextSerializerDeserializer.class.getName());

  Path tablePath = new Path(testDir, "testVariousTypes.data");
  Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(meta, schema, tablePath);
  appender.enableStats();
  appender.init();

  QueryId queryid = new QueryId("12345", 5);
  ProtobufDatumFactory factory = ProtobufDatumFactory.get(TajoIdProtos.QueryIdProto.class.getName());

  Tuple tuple = new VTuple(13);
  tuple.put(new Datum[] {
      DatumFactory.createBool(true),
      DatumFactory.createBit((byte) 0x99),
      DatumFactory.createChar("jinho"),
      DatumFactory.createInt2((short) 17),
      DatumFactory.createInt4(59),
      DatumFactory.createInt8(23l),
      DatumFactory.createFloat4(77.9f),
      DatumFactory.createFloat8(271.9f),
      DatumFactory.createText("jinho"),
      DatumFactory.createBlob("hyunsik babo".getBytes()),
      DatumFactory.createInet4("192.168.0.1"),
      NullDatum.get(),
      factory.createDatum(queryid.getProto())
  });
  appender.addTuple(tuple);
  appender.flush();
  appender.close();

  FileStatus status = fs.getFileStatus(tablePath);
  assertEquals(appender.getStats().getNumBytes().longValue(), status.getLen());

  FileFragment fragment = new FileFragment("table", tablePath, 0, status.getLen());
  Scanner scanner =  StorageManagerFactory.getStorageManager(conf).getScanner(meta, schema, fragment);
  scanner.init();

  Tuple retrieved;
  while ((retrieved=scanner.next()) != null) {
    for (int i = 0; i < tuple.size(); i++) {
      assertEquals(tuple.get(i), retrieved.get(i));
    }
  }
  scanner.close();
  assertEquals(appender.getStats().getNumBytes().longValue(), scanner.getInputStats().getNumBytes().longValue());
  assertEquals(appender.getStats().getNumRows().longValue(), scanner.getInputStats().getNumRows().longValue());
}
 
开发者ID:gruter,项目名称:tajo-cdh,代码行数:69,代码来源:TestStorages.java

示例15: testRCFileBinarySerializeDeserialize

import org.apache.tajo.datum.ProtobufDatumFactory; //导入依赖的package包/类
@Test
public void testRCFileBinarySerializeDeserialize() throws IOException {
  if(storeType != StoreType.RCFILE) return;

  Schema schema = new Schema();
  schema.addColumn("col1", Type.BOOLEAN);
  schema.addColumn("col2", Type.BIT);
  schema.addColumn("col3", Type.CHAR, 7);
  schema.addColumn("col4", Type.INT2);
  schema.addColumn("col5", Type.INT4);
  schema.addColumn("col6", Type.INT8);
  schema.addColumn("col7", Type.FLOAT4);
  schema.addColumn("col8", Type.FLOAT8);
  schema.addColumn("col9", Type.TEXT);
  schema.addColumn("col10", Type.BLOB);
  schema.addColumn("col11", Type.INET4);
  schema.addColumn("col12", Type.NULL_TYPE);
  schema.addColumn("col13", CatalogUtil.newDataType(Type.PROTOBUF, TajoIdProtos.QueryIdProto.class.getName()));

  Options options = new Options();
  TableMeta meta = CatalogUtil.newTableMeta(storeType, options);
  meta.putOption(StorageConstants.RCFILE_SERDE, BinarySerializerDeserializer.class.getName());

  Path tablePath = new Path(testDir, "testVariousTypes.data");
  Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(meta, schema, tablePath);
  appender.enableStats();
  appender.init();

  QueryId queryid = new QueryId("12345", 5);
  ProtobufDatumFactory factory = ProtobufDatumFactory.get(TajoIdProtos.QueryIdProto.class.getName());

  Tuple tuple = new VTuple(13);
  tuple.put(new Datum[] {
      DatumFactory.createBool(true),
      DatumFactory.createBit((byte) 0x99),
      DatumFactory.createChar("jinho"),
      DatumFactory.createInt2((short) 17),
      DatumFactory.createInt4(59),
      DatumFactory.createInt8(23l),
      DatumFactory.createFloat4(77.9f),
      DatumFactory.createFloat8(271.9f),
      DatumFactory.createText("jinho"),
      DatumFactory.createBlob("hyunsik babo".getBytes()),
      DatumFactory.createInet4("192.168.0.1"),
      NullDatum.get(),
      factory.createDatum(queryid.getProto())
  });
  appender.addTuple(tuple);
  appender.flush();
  appender.close();

  FileStatus status = fs.getFileStatus(tablePath);
  assertEquals(appender.getStats().getNumBytes().longValue(), status.getLen());

  FileFragment fragment = new FileFragment("table", tablePath, 0, status.getLen());
  Scanner scanner =  StorageManagerFactory.getStorageManager(conf).getScanner(meta, schema, fragment);
  scanner.init();

  Tuple retrieved;
  while ((retrieved=scanner.next()) != null) {
    for (int i = 0; i < tuple.size(); i++) {
      assertEquals(tuple.get(i), retrieved.get(i));
    }
  }
  scanner.close();
  assertEquals(appender.getStats().getNumBytes().longValue(), scanner.getInputStats().getNumBytes().longValue());
  assertEquals(appender.getStats().getNumRows().longValue(), scanner.getInputStats().getNumRows().longValue());
}
 
开发者ID:gruter,项目名称:tajo-cdh,代码行数:69,代码来源:TestStorages.java


注:本文中的org.apache.tajo.datum.ProtobufDatumFactory类示例由纯净天空整理自Github/MSDocs等开源代码及文档管理平台,相关代码片段筛选自各路编程大神贡献的开源项目,源码版权归原作者所有,传播和使用请参考对应项目的License;未经允许,请勿转载。