本文整理汇总了Java中org.apache.orc.TypeDescription.getChildren方法的典型用法代码示例。如果您正苦于以下问题:Java TypeDescription.getChildren方法的具体用法?Java TypeDescription.getChildren怎么用?Java TypeDescription.getChildren使用的例子?那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类org.apache.orc.TypeDescription
的用法示例。
在下文中一共展示了TypeDescription.getChildren方法的13个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Java代码示例。
示例1: buildSchemaDescription
import org.apache.orc.TypeDescription; //导入方法依赖的package包/类
/**
* Convert the Orc TypeDescription Object to our SchemaDescription
*
* @param typeDescription
* @return schemaDescription
*/
public SchemaDescription buildSchemaDescription( TypeDescription typeDescription ) {
SchemaDescription schemaDesc = new SchemaDescription();
Iterator fieldNameIterator = typeDescription.getFieldNames().iterator();
for ( TypeDescription subDescription : typeDescription.getChildren() ) {
//Assume getFieldNames is 1:1 with getChildren
String fieldName = (String) fieldNameIterator.next();
int metaType = determineMetaType( subDescription );
if ( metaType == -1 ) {
throw new IllegalStateException(
"Orc Field Name: " + fieldName + " - Could not find pdi field type for " + subDescription.getCategory()
.getName() );
}
schemaDesc.addField( schemaDesc.new Field( fieldName, fieldName + "",
determineMetaType( subDescription ), true ) );
}
return schemaDesc;
}
示例2: StructTreeWriter
import org.apache.orc.TypeDescription; //导入方法依赖的package包/类
StructTreeWriter(int columnId, TypeDescription schema, StreamFactory writer, boolean nullable)
throws IOException {
super(columnId, schema, writer, nullable);
List<TypeDescription> children = schema.getChildren();
childrenWriters = new TreeWriter[children.size()];
for (int i = 0; i < childrenWriters.length; ++i) {
childrenWriters[i] = createTreeWriter(children.get(i), writer, true);
}
recordPosition(rowIndexPosition);
}
示例3: MapTreeWriter
import org.apache.orc.TypeDescription; //导入方法依赖的package包/类
MapTreeWriter(int columnId, TypeDescription schema, StreamFactory writer, boolean nullable)
throws IOException {
super(columnId, schema, writer, nullable);
this.isDirectV2 = isNewWriteFormat(writer);
childrenWriters = new TreeWriter[2];
List<TypeDescription> children = schema.getChildren();
childrenWriters[0] = createTreeWriter(children.get(0), writer, true);
childrenWriters[1] = createTreeWriter(children.get(1), writer, true);
lengths = createIntegerWriter(writer.createStream(columnId, OrcProto.Stream.Kind.LENGTH),
false, isDirectV2, writer);
recordPosition(rowIndexPosition);
}
示例4: UnionTreeWriter
import org.apache.orc.TypeDescription; //导入方法依赖的package包/类
UnionTreeWriter(int columnId, TypeDescription schema, StreamFactory writer, boolean nullable)
throws IOException {
super(columnId, schema, writer, nullable);
List<TypeDescription> children = schema.getChildren();
childrenWriters = new TreeWriter[children.size()];
for (int i = 0; i < childrenWriters.length; ++i) {
childrenWriters[i] = createTreeWriter(children.get(i), writer, true);
}
tags = new RunLengthByteWriter(writer.createStream(columnId, OrcProto.Stream.Kind.DATA));
recordPosition(rowIndexPosition);
}
示例5: getStruct
import org.apache.orc.TypeDescription; //导入方法依赖的package包/类
private static Object getStruct(ColumnVector[] fields, int index, TypeDescription type) {
Dictionary<String, Object> items = new Hashtable<>();
List<String> keys = type.getFieldNames();
List<TypeDescription> children = type.getChildren();
for (int i = 0; i < fields.length; i++) {
items.put(
keys.get(i),
getValue(fields[i], index, children.get(i))
);
}
return items;
}
示例6: fillRows
import org.apache.orc.TypeDescription; //导入方法依赖的package包/类
/**
* Fills an ORC batch into an array of Row.
*
* @param rows The batch of rows need to be filled.
* @param schema The schema of the ORC data.
* @param batch The ORC data.
* @param selectedFields The list of selected ORC fields.
* @return The number of rows that were filled.
*/
static int fillRows(Row[] rows, TypeDescription schema, VectorizedRowBatch batch, int[] selectedFields) {
int rowsToRead = Math.min((int) batch.count(), rows.length);
List<TypeDescription> fieldTypes = schema.getChildren();
// read each selected field
for (int rowIdx = 0; rowIdx < selectedFields.length; rowIdx++) {
int orcIdx = selectedFields[rowIdx];
readField(rows, rowIdx, fieldTypes.get(orcIdx), batch.cols[orcIdx], null, rowsToRead);
}
return rowsToRead;
}
示例7: processRow
import org.apache.orc.TypeDescription; //导入方法依赖的package包/类
public static void processRow(JSONWriter writer, VectorizedRowBatch batch,
TypeDescription schema, int row) throws JSONException {
if (schema.getCategory() == TypeDescription.Category.STRUCT) {
List<TypeDescription> fieldTypes = schema.getChildren();
List<String> fieldNames = schema.getFieldNames();
writer.object();
for (int c = 0; c < batch.cols.length; ++c) {
writer.key(fieldNames.get(c));
setValue(writer, batch.cols[c], fieldTypes.get(c), row);
}
writer.endObject();
} else {
setValue(writer, batch.cols[0], schema, row);
}
}
示例8: setStruct
import org.apache.orc.TypeDescription; //导入方法依赖的package包/类
private static void setStruct(JSONWriter writer, StructColumnVector batch,
TypeDescription schema, int row) throws JSONException {
writer.object();
List<String> fieldNames = schema.getFieldNames();
List<TypeDescription> fieldTypes = schema.getChildren();
for (int i = 0; i < fieldTypes.size(); ++i) {
writer.key(fieldNames.get(i));
setValue(writer, batch.fields[i], fieldTypes.get(i), row);
}
writer.endObject();
}
示例9: StructColumnConverter
import org.apache.orc.TypeDescription; //导入方法依赖的package包/类
public StructColumnConverter(TypeDescription schema) {
List<TypeDescription> kids = schema.getChildren();
childrenConverters = new JsonConverter[kids.size()];
for (int c = 0; c < childrenConverters.length; ++c) {
childrenConverters[c] = createConverter(kids.get(c));
}
fieldNames = schema.getFieldNames();
}
示例10: getRawDataSize
import org.apache.orc.TypeDescription; //导入方法依赖的package包/类
private long getRawDataSize(TreeWriter child, TypeDescription schema) {
long total = 0;
long numVals = child.fileStatistics.getNumberOfValues();
switch (schema.getCategory()) {
case BOOLEAN:
case BYTE:
case SHORT:
case INT:
case FLOAT:
return numVals * JavaDataModel.get().primitive1();
case LONG:
case DOUBLE:
return numVals * JavaDataModel.get().primitive2();
case STRING:
case VARCHAR:
case CHAR:
// ORC strings are converted to java Strings. so use JavaDataModel to
// compute the overall size of strings
StringColumnStatistics scs = (StringColumnStatistics) child.fileStatistics;
numVals = numVals == 0 ? 1 : numVals;
int avgStringLen = (int) (scs.getSum() / numVals);
return numVals * JavaDataModel.get().lengthForStringOfLength(avgStringLen);
case DECIMAL:
return numVals * JavaDataModel.get().lengthOfDecimal();
case DATE:
return numVals * JavaDataModel.get().lengthOfDate();
case BINARY:
// get total length of binary blob
BinaryColumnStatistics bcs = (BinaryColumnStatistics) child.fileStatistics;
return bcs.getSum();
case TIMESTAMP:
return numVals * JavaDataModel.get().lengthOfTimestamp();
case LIST:
case MAP:
case UNION:
case STRUCT: {
TreeWriter[] childWriters = child.getChildrenWriters();
List<TypeDescription> childTypes = schema.getChildren();
for (int i = 0; i < childWriters.length; ++i) {
total += getRawDataSize(childWriters[i], childTypes.get(i));
}
break;
}
default:
LOG.debug("Unknown object inspector category.");
break;
}
return total;
}
示例11: schemaToTypeInfo
import org.apache.orc.TypeDescription; //导入方法依赖的package包/类
/**
* Converts an ORC schema to a Flink TypeInformation.
*
* @param schema The ORC schema.
* @return The TypeInformation that corresponds to the ORC schema.
*/
static TypeInformation schemaToTypeInfo(TypeDescription schema) {
switch (schema.getCategory()) {
case BOOLEAN:
return BasicTypeInfo.BOOLEAN_TYPE_INFO;
case BYTE:
return BasicTypeInfo.BYTE_TYPE_INFO;
case SHORT:
return BasicTypeInfo.SHORT_TYPE_INFO;
case INT:
return BasicTypeInfo.INT_TYPE_INFO;
case LONG:
return BasicTypeInfo.LONG_TYPE_INFO;
case FLOAT:
return BasicTypeInfo.FLOAT_TYPE_INFO;
case DOUBLE:
return BasicTypeInfo.DOUBLE_TYPE_INFO;
case DECIMAL:
return BasicTypeInfo.BIG_DEC_TYPE_INFO;
case STRING:
case CHAR:
case VARCHAR:
return BasicTypeInfo.STRING_TYPE_INFO;
case DATE:
return SqlTimeTypeInfo.DATE;
case TIMESTAMP:
return SqlTimeTypeInfo.TIMESTAMP;
case BINARY:
return PrimitiveArrayTypeInfo.BYTE_PRIMITIVE_ARRAY_TYPE_INFO;
case STRUCT:
List<TypeDescription> fieldSchemas = schema.getChildren();
TypeInformation[] fieldTypes = new TypeInformation[fieldSchemas.size()];
for (int i = 0; i < fieldSchemas.size(); i++) {
fieldTypes[i] = schemaToTypeInfo(fieldSchemas.get(i));
}
String[] fieldNames = schema.getFieldNames().toArray(new String[]{});
return new RowTypeInfo(fieldTypes, fieldNames);
case LIST:
TypeDescription elementSchema = schema.getChildren().get(0);
TypeInformation<?> elementType = schemaToTypeInfo(elementSchema);
// arrays of primitive types are handled as object arrays to support null values
return ObjectArrayTypeInfo.getInfoFor(elementType);
case MAP:
TypeDescription keySchema = schema.getChildren().get(0);
TypeDescription valSchema = schema.getChildren().get(1);
TypeInformation<?> keyType = schemaToTypeInfo(keySchema);
TypeInformation<?> valType = schemaToTypeInfo(valSchema);
return new MapTypeInfo<>(keyType, valType);
case UNION:
throw new UnsupportedOperationException("UNION type is not supported yet.");
default:
throw new IllegalArgumentException("Unknown type " + schema);
}
}
示例12: readNonNullStructColumn
import org.apache.orc.TypeDescription; //导入方法依赖的package包/类
private static void readNonNullStructColumn(Object[] vals, int fieldIdx, StructColumnVector structVector, TypeDescription schema, long[] lengthVector, int childCount) {
List<TypeDescription> childrenTypes = schema.getChildren();
int numFields = childrenTypes.size();
// create a batch of Rows to read the structs
Row[] structs = new Row[childCount];
// TODO: possible improvement: reuse existing Row objects
for (int i = 0; i < childCount; i++) {
structs[i] = new Row(numFields);
}
// read struct fields
for (int i = 0; i < numFields; i++) {
readField(structs, i, childrenTypes.get(i), structVector.fields[i], null, childCount);
}
// check if the structs need to be read into lists or as single values
if (lengthVector == null) {
if (fieldIdx == -1) { // set struct as an object
System.arraycopy(structs, 0, vals, 0, childCount);
} else { // set struct as a field of Row
Row[] rows = (Row[]) vals;
for (int i = 0; i < childCount; i++) {
rows[i].setField(fieldIdx, structs[i]);
}
}
} else { // struct in a list
int offset = 0;
Row[] temp;
for (int i = 0; offset < childCount; i++) {
temp = new Row[(int) lengthVector[i]];
System.arraycopy(structs, offset, temp, 0, temp.length);
offset = offset + temp.length;
if (fieldIdx == -1) {
vals[i] = temp;
} else {
((Row) vals[i]).setField(fieldIdx, temp);
}
}
}
}
示例13: readStructColumn
import org.apache.orc.TypeDescription; //导入方法依赖的package包/类
private static void readStructColumn(Object[] vals, int fieldIdx, StructColumnVector structVector, TypeDescription schema, long[] lengthVector, int childCount) {
List<TypeDescription> childrenTypes = schema.getChildren();
int numFields = childrenTypes.size();
// create a batch of Rows to read the structs
Row[] structs = new Row[childCount];
// TODO: possible improvement: reuse existing Row objects
for (int i = 0; i < childCount; i++) {
structs[i] = new Row(numFields);
}
// read struct fields
for (int i = 0; i < numFields; i++) {
readField(structs, i, childrenTypes.get(i), structVector.fields[i], null, childCount);
}
boolean[] isNullVector = structVector.isNull;
// check if the structs need to be read into lists or as single values
if (lengthVector == null) {
if (fieldIdx == -1) { // set struct as an object
for (int i = 0; i < childCount; i++) {
if (isNullVector[i]) {
vals[i] = null;
} else {
vals[i] = structs[i];
}
}
} else { // set struct as a field of Row
Row[] rows = (Row[]) vals;
for (int i = 0; i < childCount; i++) {
if (isNullVector[i]) {
rows[i].setField(fieldIdx, null);
} else {
rows[i].setField(fieldIdx, structs[i]);
}
}
}
} else { // struct in a list
int offset = 0;
Row[] temp;
for (int i = 0; offset < childCount; i++) {
temp = new Row[(int) lengthVector[i]];
for (int j = 0; j < temp.length; j++) {
if (isNullVector[offset]) {
temp[j] = null;
} else {
temp[j] = structs[offset++];
}
}
if (fieldIdx == -1) { // set list of structs as an object
vals[i] = temp;
} else { // set list of structs as field of row
((Row) vals[i]).setField(fieldIdx, temp);
}
}
}
}