Java ColumnarSerDe类代码示例

import org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe; //导入依赖的package包/类
@Test
public void testRCText()
        throws Exception
{
    List<TestColumn> testColumns = ImmutableList.copyOf(filter(TEST_COLUMNS, testColumn -> {
        return !testColumn.getName().equals("t_struct_null") // TODO: This is a bug in the RC text reader
                && !testColumn.getName().equals("t_map_null_key_complex_key_value"); // RC file does not support complex type as key of a map
    }));

    HiveOutputFormat<?, ?> outputFormat = new RCFileOutputFormat();
    InputFormat<?, ?> inputFormat = new RCFileInputFormat<>();
    @SuppressWarnings("deprecation")
    SerDe serde = new ColumnarSerDe();
    File file = File.createTempFile("presto_test", "rc-text");
    try {
        FileSplit split = createTestFile(file.getAbsolutePath(), outputFormat, serde, null, testColumns, NUM_ROWS);
        testCursorProvider(new ColumnarTextHiveRecordCursorProvider(), split, inputFormat, serde, testColumns, NUM_ROWS);
        testCursorProvider(new GenericHiveRecordCursorProvider(), split, inputFormat, serde, testColumns, NUM_ROWS);
    }
    finally {
        //noinspection ResultOfMethodCallIgnored
        file.delete();
    }
}
 

import org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe; //导入依赖的package包/类
@Test(enabled = false)
public void testRcTextPageSource()
        throws Exception
{
    HiveOutputFormat<?, ?> outputFormat = new RCFileOutputFormat();
    InputFormat<?, ?> inputFormat = new RCFileInputFormat<>();
    @SuppressWarnings("deprecation")
    SerDe serde = new ColumnarSerDe();
    File file = File.createTempFile("presto_test", "rc-binary");
    file.delete();
    try {
        FileSplit split = createTestFile(file.getAbsolutePath(), outputFormat, serde, null, TEST_COLUMNS, NUM_ROWS);
        testPageSourceFactory(new RcFilePageSourceFactory(TYPE_MANAGER), split, inputFormat, serde, TEST_COLUMNS);
    }
    finally {
        //noinspection ResultOfMethodCallIgnored
        file.delete();
    }
}
 

import org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe; //导入依赖的package包/类
private ColumnarStruct readColumnarStruct(BytesRefArrayWritable buff, String schema) throws SerDeException {
     Pattern pcols = Pattern.compile("[a-zA-Z_0-9]*[ ]");
     List<String> types = HiveRCSchemaUtil.parseSchemaTypes(schema);
     List<String> cols = HiveRCSchemaUtil.parseSchema(pcols, schema);

     List<FieldSchema> fieldSchemaList = new ArrayList<FieldSchema>(
         cols.size());

     for (int i = 0; i < cols.size(); i++) {
         fieldSchemaList.add(new FieldSchema(cols.get(i), HiveRCSchemaUtil
             .findPigDataType(types.get(i))));
     }

     Properties props = new Properties();

     props.setProperty(Constants.LIST_COLUMNS,
         HiveRCSchemaUtil.listToString(cols));
     props.setProperty(Constants.LIST_COLUMN_TYPES,
         HiveRCSchemaUtil.listToString(types));

     Configuration hiveConf = new HiveConf(conf, SessionState.class);
     ColumnarSerDe serde = new ColumnarSerDe();
     serde.initialize(hiveConf, props);

     return (ColumnarStruct) serde.deserialize(buff);
}
 

import org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe; //导入依赖的package包/类
@Override
public Optional<HiveRecordCursor> createHiveRecordCursor(
        String clientId,
        Configuration configuration,
        ConnectorSession session,
        Path path,
        long start,
        long length,
        Properties schema,
        List<HiveColumnHandle> columns,
        List<HivePartitionKey> partitionKeys,
        TupleDomain<HiveColumnHandle> effectivePredicate,
        DateTimeZone hiveStorageTimeZone,
        TypeManager typeManager)
{
    if (!isDeserializerClass(schema, ColumnarSerDe.class)) {
        return Optional.empty();
    }

    RecordReader<?, ?> recordReader = HiveUtil.createRecordReader(configuration, path, start, length, schema, columns);

    return Optional.<HiveRecordCursor>of(new ColumnarTextHiveRecordCursor<>(
            columnarTextRecordReader(recordReader),
            length,
            schema,
            partitionKeys,
            columns,
            hiveStorageTimeZone,
            typeManager));
}
 

import org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe; //导入依赖的package包/类
public static String getDataFormat(StorageDescriptor descriptor) {
  Preconditions.checkNotNull(descriptor);

  String serde = descriptor.getSerdeInfo().getSerializationLib();
  String inputFormat = descriptor.getInputFormat();

  if (LazySimpleSerDe.class.getName().equals(serde)) {
    if (TextInputFormat.class.getName().equals(inputFormat)) {
      return BuiltinStorages.TEXT;
    } else if (SequenceFileInputFormat.class.getName().equals(inputFormat)) {
      return BuiltinStorages.SEQUENCE_FILE;
    } else {
      throw new TajoRuntimeException(new UnknownDataFormatException(inputFormat));
    }
  } else if (LazyBinarySerDe.class.getName().equals(serde)) {
    if (SequenceFileInputFormat.class.getName().equals(inputFormat)) {
      return BuiltinStorages.SEQUENCE_FILE;
    } else {
      throw new TajoRuntimeException(new UnknownDataFormatException(inputFormat));
    }
  } else if (LazyBinaryColumnarSerDe.class.getName().equals(serde) || ColumnarSerDe.class.getName().equals(serde)) {
    if (RCFileInputFormat.class.getName().equals(inputFormat)) {
      return BuiltinStorages.RCFILE;
    } else {
      throw new TajoRuntimeException(new UnknownDataFormatException(inputFormat));
    }
  } else if (ParquetHiveSerDe.class.getName().equals(serde)) {
    return BuiltinStorages.PARQUET;
  } else if (AvroSerDe.class.getName().equals(serde)) {
    return BuiltinStorages.AVRO;
  } else if (OrcSerde.class.getName().equals(serde)) {
    return BuiltinStorages.ORC;
  } else if (RegexSerDe.class.getName().equals(serde)) {
    return BuiltinStorages.REGEX;
  } else {
    throw new TajoRuntimeException(new UnknownDataFormatException(inputFormat));
  }
}
 

import org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe; //导入依赖的package包/类
@Override
   public void prepareToRead(
    @SuppressWarnings("rawtypes") RecordReader reader, PigSplit split)
    throws IOException {

this.reader = (HiveRCRecordReader) reader;

// check that the required indexes actually exist i.e. the columns that
// should be read.
// assuming this is always defined simplifies the readColumnarTuple
// logic.

int requiredIndexes[] = getRequiredColumns();
if (requiredIndexes == null) {

    int fieldLen = pigSchema.getFields().length;

    // if any the partition keys should already exist
    String[] partitionKeys = getPartitionKeys(null, null);
    if (partitionKeys != null) {
	fieldLen += partitionKeys.length;
    }

    requiredIndexes = new int[fieldLen];

    for (int i = 0; i < fieldLen; i++) {
	requiredIndexes[i] = i;
    }

    this.requiredColumns = requiredIndexes;
}

try {
    serde = new ColumnarSerDe();
    serde.initialize(hiveConf, props);
} catch (SerDeException e) {
    LOG.error(e.toString(), e);
    throw new IOException(e);
}

   }
 

import org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe; //导入依赖的package包/类
@Override
   public void prepareToRead(
    @SuppressWarnings("rawtypes") RecordReader reader, PigSplit split)
    throws IOException {

this.reader = (HiveRCRecordReader) reader;

// check that the required indexes actually exist i.e. the columns that
// should be read.
// assuming this is always defined simplifies the readColumnarTuple
// logic.

int requiredIndexes[] = getRequiredColumns();
if (requiredIndexes == null) {

    int fieldLen = pigSchema.getFields().length;

    // if any the partition keys should already exist
    String[] partitionKeys = getPartitionKeys(null, null);
    if (partitionKeys != null) {
	fieldLen = partitionKeys.length;
    }

    requiredIndexes = new int[fieldLen];

    for (int i = 0; i < fieldLen; i++) {
	requiredIndexes[i] = i;
    }

    this.requiredColumns = requiredIndexes;
}

try {
    serde = new ColumnarSerDe();
    serde.initialize(hiveConf, props);
} catch (SerDeException e) {
    LOG.error(e.toString(), e);
    throw new IOException(e);
}

   }