Java TypeDescription.getChildren方法代码示例

import org.apache.orc.TypeDescription; //导入方法依赖的package包/类
/**
 * Convert the Orc TypeDescription Object to our SchemaDescription
 *
 * @param typeDescription
 * @return schemaDescription
 */
public SchemaDescription buildSchemaDescription( TypeDescription typeDescription ) {
  SchemaDescription schemaDesc = new SchemaDescription();
  Iterator fieldNameIterator = typeDescription.getFieldNames().iterator();
  for ( TypeDescription subDescription : typeDescription.getChildren() ) {
    //Assume getFieldNames is 1:1 with getChildren
    String fieldName = (String) fieldNameIterator.next();
    int metaType = determineMetaType( subDescription );
    if ( metaType == -1 ) {
      throw new IllegalStateException(
        "Orc Field Name: " + fieldName + " - Could not find pdi field type for " + subDescription.getCategory()
          .getName() );
    }
    schemaDesc.addField( schemaDesc.new Field( fieldName, fieldName + "",
      determineMetaType( subDescription ), true ) );
  }
  return schemaDesc;
}
 

import org.apache.orc.TypeDescription; //导入方法依赖的package包/类
StructTreeWriter(int columnId, TypeDescription schema, StreamFactory writer, boolean nullable)
    throws IOException {
  super(columnId, schema, writer, nullable);
  List<TypeDescription> children = schema.getChildren();
  childrenWriters = new TreeWriter[children.size()];
  for (int i = 0; i < childrenWriters.length; ++i) {
    childrenWriters[i] = createTreeWriter(children.get(i), writer, true);
  }
  recordPosition(rowIndexPosition);
}
 

import org.apache.orc.TypeDescription; //导入方法依赖的package包/类
MapTreeWriter(int columnId, TypeDescription schema, StreamFactory writer, boolean nullable)
    throws IOException {
  super(columnId, schema, writer, nullable);
  this.isDirectV2 = isNewWriteFormat(writer);
  childrenWriters = new TreeWriter[2];
  List<TypeDescription> children = schema.getChildren();
  childrenWriters[0] = createTreeWriter(children.get(0), writer, true);
  childrenWriters[1] = createTreeWriter(children.get(1), writer, true);
  lengths = createIntegerWriter(writer.createStream(columnId, OrcProto.Stream.Kind.LENGTH),
      false, isDirectV2, writer);
  recordPosition(rowIndexPosition);
}
 

import org.apache.orc.TypeDescription; //导入方法依赖的package包/类
UnionTreeWriter(int columnId, TypeDescription schema, StreamFactory writer, boolean nullable)
    throws IOException {
  super(columnId, schema, writer, nullable);
  List<TypeDescription> children = schema.getChildren();
  childrenWriters = new TreeWriter[children.size()];
  for (int i = 0; i < childrenWriters.length; ++i) {
    childrenWriters[i] = createTreeWriter(children.get(i), writer, true);
  }
  tags = new RunLengthByteWriter(writer.createStream(columnId, OrcProto.Stream.Kind.DATA));
  recordPosition(rowIndexPosition);
}
 

import org.apache.orc.TypeDescription; //导入方法依赖的package包/类
private static Object getStruct(ColumnVector[] fields, int index, TypeDescription type) {
    Dictionary<String, Object> items = new Hashtable<>();
    List<String> keys = type.getFieldNames();
    List<TypeDescription> children = type.getChildren();

    for (int i = 0; i < fields.length; i++) {
        items.put(
                keys.get(i),
                getValue(fields[i], index, children.get(i))
        );
    }

    return items;
}
 

import org.apache.orc.TypeDescription; //导入方法依赖的package包/类
/**
 * Fills an ORC batch into an array of Row.
 *
 * @param rows The batch of rows need to be filled.
 * @param schema The schema of the ORC data.
 * @param batch The ORC data.
 * @param selectedFields The list of selected ORC fields.
 * @return The number of rows that were filled.
 */
static int fillRows(Row[] rows, TypeDescription schema, VectorizedRowBatch batch, int[] selectedFields) {

	int rowsToRead = Math.min((int) batch.count(), rows.length);

	List<TypeDescription> fieldTypes = schema.getChildren();
	// read each selected field
	for (int rowIdx = 0; rowIdx < selectedFields.length; rowIdx++) {
		int orcIdx = selectedFields[rowIdx];
		readField(rows, rowIdx, fieldTypes.get(orcIdx), batch.cols[orcIdx], null, rowsToRead);
	}
	return rowsToRead;
}
 

import org.apache.orc.TypeDescription; //导入方法依赖的package包/类
public static void processRow(JSONWriter writer, VectorizedRowBatch batch,
        TypeDescription schema, int row) throws JSONException {
    if (schema.getCategory() == TypeDescription.Category.STRUCT) {
        List<TypeDescription> fieldTypes = schema.getChildren();
        List<String> fieldNames = schema.getFieldNames();
        writer.object();
        for (int c = 0; c < batch.cols.length; ++c) {
            writer.key(fieldNames.get(c));
            setValue(writer, batch.cols[c], fieldTypes.get(c), row);
        }
        writer.endObject();
    } else {
        setValue(writer, batch.cols[0], schema, row);
    }
}
 

import org.apache.orc.TypeDescription; //导入方法依赖的package包/类
private static void setStruct(JSONWriter writer, StructColumnVector batch,
        TypeDescription schema, int row) throws JSONException {
    writer.object();
    List<String> fieldNames = schema.getFieldNames();
    List<TypeDescription> fieldTypes = schema.getChildren();
    for (int i = 0; i < fieldTypes.size(); ++i) {
        writer.key(fieldNames.get(i));
        setValue(writer, batch.fields[i], fieldTypes.get(i), row);
    }
    writer.endObject();
}
 

import org.apache.orc.TypeDescription; //导入方法依赖的package包/类
public StructColumnConverter(TypeDescription schema) {
    List<TypeDescription> kids = schema.getChildren();
    childrenConverters = new JsonConverter[kids.size()];
    for (int c = 0; c < childrenConverters.length; ++c) {
        childrenConverters[c] = createConverter(kids.get(c));
    }
    fieldNames = schema.getFieldNames();
}
 

import org.apache.orc.TypeDescription; //导入方法依赖的package包/类
private long getRawDataSize(TreeWriter child, TypeDescription schema) {
  long total = 0;
  long numVals = child.fileStatistics.getNumberOfValues();
  switch (schema.getCategory()) {
    case BOOLEAN:
    case BYTE:
    case SHORT:
    case INT:
    case FLOAT:
      return numVals * JavaDataModel.get().primitive1();
    case LONG:
    case DOUBLE:
      return numVals * JavaDataModel.get().primitive2();
    case STRING:
    case VARCHAR:
    case CHAR:
      // ORC strings are converted to java Strings. so use JavaDataModel to
      // compute the overall size of strings
      StringColumnStatistics scs = (StringColumnStatistics) child.fileStatistics;
      numVals = numVals == 0 ? 1 : numVals;
      int avgStringLen = (int) (scs.getSum() / numVals);
      return numVals * JavaDataModel.get().lengthForStringOfLength(avgStringLen);
    case DECIMAL:
      return numVals * JavaDataModel.get().lengthOfDecimal();
    case DATE:
      return numVals * JavaDataModel.get().lengthOfDate();
    case BINARY:
      // get total length of binary blob
      BinaryColumnStatistics bcs = (BinaryColumnStatistics) child.fileStatistics;
      return bcs.getSum();
    case TIMESTAMP:
      return numVals * JavaDataModel.get().lengthOfTimestamp();
    case LIST:
    case MAP:
    case UNION:
    case STRUCT: {
      TreeWriter[] childWriters = child.getChildrenWriters();
      List<TypeDescription> childTypes = schema.getChildren();
      for (int i = 0; i < childWriters.length; ++i) {
        total += getRawDataSize(childWriters[i], childTypes.get(i));
      }
      break;
    }
    default:
      LOG.debug("Unknown object inspector category.");
      break;
  }
  return total;
}
 

import org.apache.orc.TypeDescription; //导入方法依赖的package包/类
/**
 * Converts an ORC schema to a Flink TypeInformation.
 *
 * @param schema The ORC schema.
 * @return The TypeInformation that corresponds to the ORC schema.
 */
static TypeInformation schemaToTypeInfo(TypeDescription schema) {
	switch (schema.getCategory()) {
		case BOOLEAN:
			return BasicTypeInfo.BOOLEAN_TYPE_INFO;
		case BYTE:
			return BasicTypeInfo.BYTE_TYPE_INFO;
		case SHORT:
			return BasicTypeInfo.SHORT_TYPE_INFO;
		case INT:
			return BasicTypeInfo.INT_TYPE_INFO;
		case LONG:
			return BasicTypeInfo.LONG_TYPE_INFO;
		case FLOAT:
			return BasicTypeInfo.FLOAT_TYPE_INFO;
		case DOUBLE:
			return BasicTypeInfo.DOUBLE_TYPE_INFO;
		case DECIMAL:
			return BasicTypeInfo.BIG_DEC_TYPE_INFO;
		case STRING:
		case CHAR:
		case VARCHAR:
			return BasicTypeInfo.STRING_TYPE_INFO;
		case DATE:
			return SqlTimeTypeInfo.DATE;
		case TIMESTAMP:
			return SqlTimeTypeInfo.TIMESTAMP;
		case BINARY:
			return PrimitiveArrayTypeInfo.BYTE_PRIMITIVE_ARRAY_TYPE_INFO;
		case STRUCT:
			List<TypeDescription> fieldSchemas = schema.getChildren();
			TypeInformation[] fieldTypes = new TypeInformation[fieldSchemas.size()];
			for (int i = 0; i < fieldSchemas.size(); i++) {
				fieldTypes[i] = schemaToTypeInfo(fieldSchemas.get(i));
			}
			String[] fieldNames = schema.getFieldNames().toArray(new String[]{});
			return new RowTypeInfo(fieldTypes, fieldNames);
		case LIST:
			TypeDescription elementSchema = schema.getChildren().get(0);
			TypeInformation<?> elementType = schemaToTypeInfo(elementSchema);
			// arrays of primitive types are handled as object arrays to support null values
			return ObjectArrayTypeInfo.getInfoFor(elementType);
		case MAP:
			TypeDescription keySchema = schema.getChildren().get(0);
			TypeDescription valSchema = schema.getChildren().get(1);
			TypeInformation<?> keyType = schemaToTypeInfo(keySchema);
			TypeInformation<?> valType = schemaToTypeInfo(valSchema);
			return new MapTypeInfo<>(keyType, valType);
		case UNION:
			throw new UnsupportedOperationException("UNION type is not supported yet.");
		default:
			throw new IllegalArgumentException("Unknown type " + schema);
	}
}
 

import org.apache.orc.TypeDescription; //导入方法依赖的package包/类
private static void readNonNullStructColumn(Object[] vals, int fieldIdx, StructColumnVector structVector, TypeDescription schema, long[] lengthVector, int childCount) {

		List<TypeDescription> childrenTypes = schema.getChildren();

		int numFields = childrenTypes.size();
		// create a batch of Rows to read the structs
		Row[] structs = new Row[childCount];
		// TODO: possible improvement: reuse existing Row objects
		for (int i = 0; i < childCount; i++) {
			structs[i] = new Row(numFields);
		}

		// read struct fields
		for (int i = 0; i < numFields; i++) {
			readField(structs, i, childrenTypes.get(i), structVector.fields[i], null, childCount);
		}

		// check if the structs need to be read into lists or as single values
		if (lengthVector == null) {
			if (fieldIdx == -1) { // set struct as an object
				System.arraycopy(structs, 0, vals, 0, childCount);
			} else { // set struct as a field of Row
				Row[] rows = (Row[]) vals;
				for (int i = 0; i < childCount; i++) {
					rows[i].setField(fieldIdx, structs[i]);
				}
			}
		} else { // struct in a list
			int offset = 0;
			Row[] temp;
			for (int i = 0; offset < childCount; i++) {
				temp = new Row[(int) lengthVector[i]];
				System.arraycopy(structs, offset, temp, 0, temp.length);
				offset = offset + temp.length;
				if (fieldIdx == -1) {
					vals[i] = temp;
				} else {
					((Row) vals[i]).setField(fieldIdx, temp);
				}
			}
		}
	}
 

import org.apache.orc.TypeDescription; //导入方法依赖的package包/类
private static void readStructColumn(Object[] vals, int fieldIdx, StructColumnVector structVector, TypeDescription schema, long[] lengthVector, int childCount) {

		List<TypeDescription> childrenTypes = schema.getChildren();

		int numFields = childrenTypes.size();
		// create a batch of Rows to read the structs
		Row[] structs = new Row[childCount];
		// TODO: possible improvement: reuse existing Row objects
		for (int i = 0; i < childCount; i++) {
			structs[i] = new Row(numFields);
		}

		// read struct fields
		for (int i = 0; i < numFields; i++) {
			readField(structs, i, childrenTypes.get(i), structVector.fields[i], null, childCount);
		}

		boolean[] isNullVector = structVector.isNull;

		// check if the structs need to be read into lists or as single values
		if (lengthVector == null) {
			if (fieldIdx == -1) { // set struct as an object
				for (int i = 0; i < childCount; i++) {
					if (isNullVector[i]) {
						vals[i] = null;
					} else {
						vals[i] = structs[i];
					}
				}
			} else { // set struct as a field of Row
				Row[] rows = (Row[]) vals;
				for (int i = 0; i < childCount; i++) {
					if (isNullVector[i]) {
						rows[i].setField(fieldIdx, null);
					} else {
						rows[i].setField(fieldIdx, structs[i]);
					}
				}
			}
		} else { // struct in a list
			int offset = 0;
			Row[] temp;
			for (int i = 0; offset < childCount; i++) {
				temp = new Row[(int) lengthVector[i]];
				for (int j = 0; j < temp.length; j++) {
					if (isNullVector[offset]) {
						temp[j] = null;
					} else {
						temp[j] = structs[offset++];
					}
				}
				if (fieldIdx == -1) { // set list of structs as an object
					vals[i] = temp;
				} else { // set list of structs as field of row
					((Row) vals[i]).setField(fieldIdx, temp);
				}
			}
		}
	}