Java LogicalTypes类代码示例

import org.apache.avro.LogicalTypes; //导入依赖的package包/类
@Test
public void toTypeSchemaDateNullable() throws Exception {
  Schema schema = AvroUtils.typeFor(DataTypes.DateType);

  assertEquals("Invalid type", Schema.Type.UNION, schema.getType());
  assertEquals("Invalid union size", 2, schema.getTypes().size());

  for (Schema s : schema.getTypes()) {
    assertThat("Invalid union types", s.getType(), anyOf(is(Schema.Type.INT), is(Schema.Type.NULL)));
    if (s.getType().equals(Schema.Type.INT)) {
      assertEquals("Invalid LogicalType", LogicalTypes.date(), s.getLogicalType());
    }
  }

  //System.out.println(schema.toString(true));
}
 

import org.apache.avro.LogicalTypes; //导入依赖的package包/类
@Test
public void toTypeSchemaTimestampNullable() throws Exception {
  Schema schema = AvroUtils.typeFor(DataTypes.TimestampType);

  assertEquals("Invalid type", Schema.Type.UNION, schema.getType());
  assertEquals("Invalid union size", 2, schema.getTypes().size());

  for (Schema s : schema.getTypes()) {
    assertThat("Invalid union types", s.getType(), anyOf(is(Schema.Type.LONG), is(Schema.Type.NULL)));
    if (s.getType().equals(Schema.Type.LONG)) {
      assertEquals("Invalid LogicalType", LogicalTypes.timestampMillis(), s.getLogicalType());
    }
  }

  //System.out.println(schema.toString(true));
}
 

import org.apache.avro.LogicalTypes; //导入依赖的package包/类
@Test
public void toTypeSchemaDecimalNullable() throws Exception {
  // Defaults to precision 10, scale 0
  Schema schema = AvroUtils.typeFor(DataTypes.createDecimalType());

  assertEquals("Invalid type", Schema.Type.UNION, schema.getType());
  assertEquals("Invalid union size", 2, schema.getTypes().size());

  for (Schema s : schema.getTypes()) {
    assertThat("Invalid union types", s.getType(), anyOf(is(Schema.Type.BYTES), is(Schema.Type.NULL)));
    if (s.getType().equals(Schema.Type.BYTES)) {
      assertEquals("Invalid LogicalType", LogicalTypes.decimal(10, 0), s.getLogicalType());
    }
  }

  //System.out.println(schema.toString(true));
}
 

import org.apache.avro.LogicalTypes; //导入依赖的package包/类
/**
 * Generate a schema from output type and format
 *
 * @param outputType
 * @param outputFormat
 * @return
 */
public static Schema getSchema(TypeConverterProperties.TypeConverterOutputTypes outputType, String outputFormat) {
    Schema result = Schema.create(outputType.getTargetType());
    switch (outputType) {
        case Decimal:
            result = LogicalTypes.decimal(20, 4).addToSchema(result);
            break;
        case Date:
            result = LogicalTypes.date().addToSchema(result);
            break;
        case Time:
            result = LogicalTypes.timeMillis().addToSchema(result);
            break;
        case DateTime:
            result = LogicalTypes.timestampMillis().addToSchema(result);
            break;
    }
    return result;
}
 

import org.apache.avro.LogicalTypes; //导入依赖的package包/类
private OriginalType convertLogicalType(LogicalType logicalType) {
  if (logicalType == null) {
    return null;
  } else if (logicalType instanceof LogicalTypes.Decimal) {
    return OriginalType.DECIMAL;
  } else if (logicalType instanceof LogicalTypes.Date) {
    return OriginalType.DATE;
  } else if (logicalType instanceof LogicalTypes.TimeMillis) {
    return OriginalType.TIME_MILLIS;
  } else if (logicalType instanceof LogicalTypes.TimeMicros) {
    return OriginalType.TIME_MICROS;
  } else if (logicalType instanceof LogicalTypes.TimestampMillis) {
    return OriginalType.TIMESTAMP_MILLIS;
  } else if (logicalType instanceof LogicalTypes.TimestampMicros) {
    return OriginalType.TIMESTAMP_MICROS;
  }
  return null;
}
 

import org.apache.avro.LogicalTypes; //导入依赖的package包/类
private LogicalType convertOriginalType(OriginalType annotation, DecimalMetadata meta) {
  if (annotation == null) {
    return null;
  }
  switch (annotation) {
    case DECIMAL:
      return LogicalTypes.decimal(meta.getPrecision(), meta.getScale());
    case DATE:
      return LogicalTypes.date();
    case TIME_MILLIS:
      return LogicalTypes.timeMillis();
    case TIME_MICROS:
      return LogicalTypes.timeMicros();
    case TIMESTAMP_MILLIS:
      return LogicalTypes.timestampMillis();
    case TIMESTAMP_MICROS:
      return LogicalTypes.timestampMicros();
  }
  return null;
}
 

import org.apache.avro.LogicalTypes; //导入依赖的package包/类
@Test
public void testDecimalBytes() throws IOException {
  Schema schema = REFLECT.getSchema(DecimalRecordBytes.class);
  Assert.assertEquals("Should have the correct record name",
      "org.apache.parquet.avro.TestReflectLogicalTypes$",
      schema.getNamespace());
  Assert.assertEquals("Should have the correct record name",
      "DecimalRecordBytes",
      schema.getName());
  Assert.assertEquals("Should have the correct logical type",
      LogicalTypes.decimal(9, 2),
      LogicalTypes.fromSchema(schema.getField("decimal").schema()));

  DecimalRecordBytes record = new DecimalRecordBytes();
  record.decimal = new BigDecimal("3.14");

  File test = write(REFLECT, schema, record);
  Assert.assertEquals("Should match the decimal after round trip",
      Arrays.asList(record),
      read(REFLECT, schema, test));
}
 

import org.apache.avro.LogicalTypes; //导入依赖的package包/类
@Test
public void testDecimalFixed() throws IOException {
  Schema schema = REFLECT.getSchema(DecimalRecordFixed.class);
  Assert.assertEquals("Should have the correct record name",
      "org.apache.parquet.avro.TestReflectLogicalTypes$",
      schema.getNamespace());
  Assert.assertEquals("Should have the correct record name",
      "DecimalRecordFixed",
      schema.getName());
  Assert.assertEquals("Should have the correct logical type",
      LogicalTypes.decimal(9, 2),
      LogicalTypes.fromSchema(schema.getField("decimal").schema()));

  DecimalRecordFixed record = new DecimalRecordFixed();
  record.decimal = new BigDecimal("3.14");

  File test = write(REFLECT, schema, record);
  Assert.assertEquals("Should match the decimal after round trip",
      Arrays.asList(record),
      read(REFLECT, schema, test));
}
 

import org.apache.avro.LogicalTypes; //导入依赖的package包/类
@Test(expected = ClassCastException.class)
public void testWriteUUIDMissingLogicalType() throws IOException {
  Schema uuidSchema = SchemaBuilder.record(RecordWithUUID.class.getName())
      .fields().requiredString("uuid").endRecord();
  LogicalTypes.uuid().addToSchema(uuidSchema.getField("uuid").schema());

  UUID u1 = UUID.randomUUID();
  UUID u2 = UUID.randomUUID();

  RecordWithUUID r1 = new RecordWithUUID();
  r1.uuid = u1;
  RecordWithUUID r2 = new RecordWithUUID();
  r2.uuid = u2;

  // write without using REFLECT, which has the logical type
  File test = write(uuidSchema, r1, r2);

  // verify that the field's type overrides the logical type
  Schema uuidStringSchema = SchemaBuilder
      .record(RecordWithStringUUID.class.getName())
      .fields().requiredString("uuid").endRecord();

  // this fails with an AppendWriteException wrapping ClassCastException
  // because the UUID isn't converted to a CharSequence expected internally
  read(ReflectData.get(), uuidStringSchema, test);
}
 

import org.apache.avro.LogicalTypes; //导入依赖的package包/类
@Test
public void testReadUUIDArray() throws IOException {
  Schema uuidArraySchema = SchemaBuilder.record(RecordWithUUIDArray.class.getName())
      .fields()
      .name("uuids").type().array().items().stringType().noDefault()
      .endRecord();
  LogicalTypes.uuid().addToSchema(
      uuidArraySchema.getField("uuids").schema().getElementType());

  UUID u1 = UUID.randomUUID();
  UUID u2 = UUID.randomUUID();

  GenericRecord r = new GenericData.Record(uuidArraySchema);
  r.put("uuids", Arrays.asList(u1.toString(), u2.toString()));

  RecordWithUUIDArray expected = new RecordWithUUIDArray();
  expected.uuids = new UUID[] {u1, u2};

  File test = write(uuidArraySchema, r);

  Assert.assertEquals("Should convert Strings to UUIDs",
      expected,
      read(REFLECT, uuidArraySchema, test).get(0));
}
 

import org.apache.avro.LogicalTypes; //导入依赖的package包/类
@Test
public void testReadUUIDList() throws IOException {
  Schema uuidListSchema = SchemaBuilder.record(RecordWithUUIDList.class.getName())
      .fields()
      .name("uuids").type().array().items().stringType().noDefault()
      .endRecord();
  uuidListSchema.getField("uuids").schema().addProp(
      SpecificData.CLASS_PROP, List.class.getName());
  LogicalTypes.uuid().addToSchema(
      uuidListSchema.getField("uuids").schema().getElementType());

  UUID u1 = UUID.randomUUID();
  UUID u2 = UUID.randomUUID();

  GenericRecord r = new GenericData.Record(uuidListSchema);
  r.put("uuids", Arrays.asList(u1.toString(), u2.toString()));

  RecordWithUUIDList expected = new RecordWithUUIDList();
  expected.uuids = Arrays.asList(u1, u2);

  File test = write(uuidListSchema, r);

  Assert.assertEquals("Should convert Strings to UUIDs",
      expected, read(REFLECT, uuidListSchema, test).get(0));
}
 

import org.apache.avro.LogicalTypes; //导入依赖的package包/类
@Test
public void testWriteUUIDReadStringSchema() throws IOException {
  Schema uuidSchema = record("R",
      field("uuid", LogicalTypes.uuid().addToSchema(Schema.create(STRING))));
  GenericRecord u1 = instance(uuidSchema, "uuid", UUID.randomUUID());
  GenericRecord u2 = instance(uuidSchema, "uuid", UUID.randomUUID());

  Schema stringUuidSchema = Schema.create(STRING);
  stringUuidSchema.addProp(GenericData.STRING_PROP, "String");
  Schema stringSchema = record("R", field("uuid", stringUuidSchema));
  GenericRecord s1 = instance(stringSchema, "uuid", u1.get("uuid").toString());
  GenericRecord s2 = instance(stringSchema, "uuid", u2.get("uuid").toString());

  File test = write(GENERIC, uuidSchema, u1, u2);
  Assert.assertEquals("Should read UUIDs as Strings",
      Arrays.asList(s1, s2), read(GENERIC, stringSchema, test));
}
 

import org.apache.avro.LogicalTypes; //导入依赖的package包/类
@Test
public void testWriteNullableUUID() throws IOException {
  Schema nullableUuidSchema = record("R",
      optionalField("uuid", LogicalTypes.uuid().addToSchema(Schema.create(STRING))));
  GenericRecord u1 = instance(nullableUuidSchema, "uuid", UUID.randomUUID());
  GenericRecord u2 = instance(nullableUuidSchema, "uuid", UUID.randomUUID());

  Schema stringUuidSchema = Schema.create(STRING);
  stringUuidSchema.addProp(GenericData.STRING_PROP, "String");
  Schema nullableStringSchema = record("R", optionalField("uuid", stringUuidSchema));
  GenericRecord s1 = instance(nullableStringSchema, "uuid", u1.get("uuid").toString());
  GenericRecord s2 = instance(nullableStringSchema, "uuid", u2.get("uuid").toString());

  File test = write(GENERIC, nullableUuidSchema, u1, u2);
  Assert.assertEquals("Should read UUIDs as Strings",
      Arrays.asList(s1, s2), read(GENERIC, nullableStringSchema, test));
}
 

import org.apache.avro.LogicalTypes; //导入依赖的package包/类
private static Schema typeFor(DataType dataType, boolean isOptional, int recordCount) {
  LOG.trace("Converting {} to Avro, optional[{}]", dataType, isOptional);

  Schema typeSchema;
  SchemaBuilder.BaseTypeBuilder<Schema> typeBuilder = SchemaBuilder.builder();

  switch (dataType.typeName()) {
    case "binary":
      // bytes
      typeSchema = typeBuilder.bytesType();
      break;
    case "boolean":
      typeSchema = typeBuilder.booleanType();
      break;
    case "date":
      // int (logical)
      typeSchema = LogicalTypes.date().addToSchema(typeBuilder.intType());
      break;
    case "timestamp":
      // long (logical)
      typeSchema = LogicalTypes.timestampMillis().addToSchema(typeBuilder.longType());
      break;
    case "double":
      typeSchema = typeBuilder.doubleType();
      break;
    case "float":
      typeSchema = typeBuilder.floatType();
      break;
    case "integer":
    case "byte":
    case "short":
      typeSchema = typeBuilder.intType();
      break;
    case "long":
      typeSchema = typeBuilder.longType();
      break;
    case "null":
      typeSchema = typeBuilder.nullType();
      break;
    case "string":
      typeSchema = typeBuilder.stringType();
      break;
    case "array":
      ArrayType arrayType = (ArrayType) dataType;
      typeSchema = typeBuilder.array().items(typeFor(arrayType.elementType(), arrayType.containsNull(), recordCount));
      break;
    case "map":
      MapType mapType = (MapType) dataType;
      // Keys must be strings: mapType.keyType()
      typeSchema = typeBuilder.map().values(typeFor(mapType.valueType(), mapType.valueContainsNull(), recordCount));
      break;
    case "struct":
      StructType structType = (StructType) dataType;
      // Nested "anonymous" records
      typeSchema = schemaFor(structType, null, null, recordCount);
      break;
    default:
      if (dataType.typeName().startsWith("decimal")) {
        // byte (logical)
        DecimalType decimalType = (DecimalType) dataType;
        typeSchema = LogicalTypes.decimal(decimalType.precision(), decimalType.scale()).addToSchema(typeBuilder.bytesType());
      } else {
        throw new RuntimeException(String.format("DataType[%s] - DataType unrecognized or not yet implemented",
            dataType));
      }
  }

  if (isOptional && !typeSchema.getType().equals(Schema.Type.NULL)) {
    return SchemaBuilder.builder().nullable().type(typeSchema);
  }

  return typeSchema;
}
 

import org.apache.avro.LogicalTypes; //导入依赖的package包/类
@Test
public void toTypeSchemaDateNotNullable() throws Exception {
  Schema schema = AvroUtils.typeFor(DataTypes.DateType, false);

  assertEquals("Invalid type", Schema.Type.INT, schema.getType());
  assertEquals("Invalid LogicalType", LogicalTypes.date(), schema.getLogicalType());

  //System.out.println(schema.toString(true));
}