Java StructType.fields方法代码示例

import org.apache.spark.sql.types.StructType; //导入方法依赖的package包/类
private static Schema schemaFor(StructType structType, String record, String namespace, int recordCount) {

    // Increment after using the recordCount
    record = (null == record) ? "record" + recordCount++ : record;

    LOG.debug("Converting {} to Avro Record schema [{}:{}]", structType, record, namespace);
    SchemaBuilder.RecordBuilder<Schema> schema = SchemaBuilder.record(record);

    if (null != namespace) {
      schema.namespace(namespace);
    }

    schema.doc("Auto-generated from Spark DataFrame");

    SchemaBuilder.FieldAssembler<Schema> assembler = schema.fields();
    StructField[] structFields = structType.fields();

    for (StructField f : structFields) {
      assembler.name(f.name()).type(typeFor(f.dataType(), f.nullable(), recordCount)).noDefault();
    }

    return assembler.endRecord();
  }
 

import org.apache.spark.sql.types.StructType; //导入方法依赖的package包/类
@Override
public StructType transformSchema(StructType schema){
	StructType structSchema = getStructSchema(schema);

	StructType result = schema;

	StructField[] fields = structSchema.fields();
	for(StructField field : fields){
		result = result.add(field);
	}

	return result;
}
 

import org.apache.spark.sql.types.StructType; //导入方法依赖的package包/类
/**
 * Create a datavec schema
 * from a struct type
 *
 * @param structType the struct type to create the schema from
 * @return the created schema
 */
public static Schema fromStructType(StructType structType) {
    Schema.Builder builder = new Schema.Builder();
    StructField[] fields = structType.fields();
    String[] fieldNames = structType.fieldNames();
    for (int i = 0; i < fields.length; i++) {
        String name = fields[i].dataType().typeName().toLowerCase();
        switch (name) {
            case "double":
                builder.addColumnDouble(fieldNames[i]);
                break;
            case "float":
                builder.addColumnFloat(fieldNames[i]);
                break;
            case "long":
                builder.addColumnLong(fieldNames[i]);
                break;
            case "int":
            case "integer":
                builder.addColumnInteger(fieldNames[i]);
                break;
            case "string":
                builder.addColumnString(fieldNames[i]);
                break;
            default:
                throw new RuntimeException("Unknown type: " + name);
        }
    }

    return builder.build();
}
 

import org.apache.spark.sql.types.StructType; //导入方法依赖的package包/类
/**
 * @return true if UnsafeFixedWidthAggregationMap supports aggregation buffers with the given
 * schema, false otherwise.
 */
public static boolean supportsAggregationBufferSchema(StructType schema) {
    for (StructField field : schema.fields()) {
        if (!UnsafeRow.isMutable(field.dataType())) {
            return false;
        }
    }
    return true;
}
 

import org.apache.spark.sql.types.StructType; //导入方法依赖的package包/类
public static StructType subtractSchema(StructType schema, List<String> subtractFieldNames) {
  List<String> fieldNames = Lists.newArrayList();
  
  for (StructField schemaField : schema.fields()) {
    if (!subtractFieldNames.contains(schemaField.name())) {
      fieldNames.add(schemaField.name());
    }
  }
  
  StructType subtracted = subsetSchema(schema, fieldNames);
  
  return subtracted;
}
 

import org.apache.spark.sql.types.StructType; //导入方法依赖的package包/类
@Test
public void testBuildSchema() {
  Config goodConfig = ConfigUtils.configFromResource("/hbase/hbase-output.conf").getConfig("output");

  Map<String, HBaseSerde.ColumnDef> columnDefs = HBaseUtils.columnsFor(goodConfig);
  StructType schema = HBaseUtils.buildSchema(columnDefs);

  Map<String, DataType> shouldHaveFields = Maps.newHashMap();
  shouldHaveFields.put("symbol", DataTypes.StringType);
  shouldHaveFields.put("transacttime", DataTypes.LongType);
  shouldHaveFields.put("clordid", DataTypes.StringType);
  shouldHaveFields.put("orderqty", DataTypes.IntegerType);
  shouldHaveFields.put("leavesqty", DataTypes.IntegerType);
  shouldHaveFields.put("cumqty", DataTypes.IntegerType);

  StructField[] fields = schema.fields();
  assertEquals("Schema should contain 6 fields", 6, fields.length);

  Map<String, DataType> hasFields = Maps.newHashMap();
  for (StructField field : fields) {
    hasFields.put(field.name(), field.dataType());
  }

  for (Map.Entry<String, DataType> shouldHave : shouldHaveFields.entrySet()) {
    assertTrue("Schema should have field: " + shouldHave.getKey(),
        hasFields.containsKey(shouldHave.getKey()));
    assertEquals("Field " + shouldHave.getKey() + " should have type: " +
            shouldHave.getValue(), shouldHave.getValue(),
        hasFields.get(shouldHave.getKey()));
  }

}
 

import org.apache.spark.sql.types.StructType; //导入方法依赖的package包/类
@Override
public Dataset<Row> transform(Dataset<?> dataset){
	StructType schema = dataset.schema();

	StructType structSchema = getStructSchema(schema);

	Column structColumn = dataset.apply(DatasetUtil.escapeColumnName(getStructCol()));

	Dataset<Row> result = dataset.toDF();

	StructField[] fields = structSchema.fields();
	for(StructField field : fields){
		String name = field.name();

		Column fieldColumn = structColumn.getField(DatasetUtil.escapeColumnName(name));

		result = result.withColumn(DatasetUtil.escapeColumnName(name), fieldColumn);
	}

	return result;
}