Java Schema.createRecord方法代码示例

import org.apache.avro.Schema; //导入方法依赖的package包/类
@Test
public void testValidateSchemaPairAllFields() throws Exception {
  final List<Schema.Field> readerFields = list(
      new Schema.Field("oldfield1", INT_SCHEMA, null, null),
      new Schema.Field("oldfield2", STRING_SCHEMA, null, null));
  final Schema reader = Schema.createRecord(readerFields);
  final SchemaCompatibility.SchemaPairCompatibility expectedResult =
      new SchemaCompatibility.SchemaPairCompatibility(
          SchemaCompatibility.SchemaCompatibilityResult.compatible(),
          reader,
          WRITER_SCHEMA,
          SchemaCompatibility.READER_WRITER_COMPATIBLE_MESSAGE);

  // Test with all fields.
  assertEquals(expectedResult, checkReaderWriterCompatibility(reader, WRITER_SCHEMA));
}
 

import org.apache.avro.Schema; //导入方法依赖的package包/类
@Test
public void testValidateSchemaNewFieldWithDefault() throws Exception {
  final List<Schema.Field> readerFields = list(
      new Schema.Field("oldfield1", INT_SCHEMA, null, null),
      new Schema.Field("newfield1", INT_SCHEMA, null, 42));
  final Schema reader = Schema.createRecord(readerFields);
  final SchemaCompatibility.SchemaPairCompatibility expectedResult =
      new SchemaCompatibility.SchemaPairCompatibility(
          SchemaCompatibility.SchemaCompatibilityResult.compatible(),
          reader,
          WRITER_SCHEMA,
          SchemaCompatibility.READER_WRITER_COMPATIBLE_MESSAGE);

  // Test new field with default value.
  assertEquals(expectedResult, checkReaderWriterCompatibility(reader, WRITER_SCHEMA));
}
 

import org.apache.avro.Schema; //导入方法依赖的package包/类
@Test
public void testValidateSchemaNewField() throws Exception {
  final List<Schema.Field> readerFields = list(
      new Schema.Field("oldfield1", INT_SCHEMA, null, null),
      new Schema.Field("newfield1", INT_SCHEMA, null, null));
  final Schema reader = Schema.createRecord(readerFields);
  SchemaPairCompatibility compatibility = checkReaderWriterCompatibility(reader, WRITER_SCHEMA);

  // Test new field without default value.
  assertEquals(SchemaCompatibility.SchemaCompatibilityType.INCOMPATIBLE, compatibility.getType());
  assertEquals(SchemaCompatibility.SchemaCompatibilityResult.incompatible(SchemaIncompatibilityType.READER_FIELD_MISSING_DEFAULT_VALUE, reader, WRITER_SCHEMA, "newfield1", Arrays.asList("", "fields", "1")), compatibility.getResult());
  assertEquals(String.format("Data encoded using writer schema:%n%s%n"
      + "will or may fail to decode using reader schema:%n%s%n",
      WRITER_SCHEMA.toString(true),
      reader.toString(true)), compatibility.getDescription());
  assertEquals(reader, compatibility.getReader());
  assertEquals(WRITER_SCHEMA, compatibility.getWriter());
}
 

import org.apache.avro.Schema; //导入方法依赖的package包/类
public void testParquetRecordsNotSupported() throws IOException, SQLException {
  String[] argv = {};
  final int TOTAL_RECORDS = 1;

  Schema schema =  Schema.createRecord("nestedrecord", null, null, false);
  schema.setFields(Lists.newArrayList(buildField("myint",
      Schema.Type.INT)));
  GenericRecord record = new GenericData.Record(schema);
  record.put("myint", 100);
  // DB type is not used so can be anything:
  ColumnGenerator gen = colGenerator(record, schema, null, "VARCHAR(64)");
  createParquetFile(0, TOTAL_RECORDS,  gen);
  createTable(gen);
  try {
    runExport(getArgv(true, 10, 10, newStrArray(argv, "-m", "" + 1)));
    fail("Parquet records can not be exported.");
  } catch (Exception e) {
    // expected
    assertTrue(true);
  }
}
 

import org.apache.avro.Schema; //导入方法依赖的package包/类
public void testAvroRecordsNotSupported() throws IOException, SQLException {
  String[] argv = {};
  final int TOTAL_RECORDS = 1;

  Schema schema =  Schema.createRecord("nestedrecord", null, null, false);
  schema.setFields(Lists.newArrayList(buildAvroField("myint",
      Schema.Type.INT)));
  GenericRecord record = new GenericData.Record(schema);
  record.put("myint", 100);
  // DB type is not used so can be anything:
  ColumnGenerator gen = colGenerator(record, schema, null, "VARCHAR(64)");
  createAvroFile(0, TOTAL_RECORDS,  gen);
  createTable(gen);
  try {
    runExport(getArgv(true, 10, 10, newStrArray(argv, "-m", "" + 1)));
    fail("Avro records can not be exported.");
  } catch (Exception e) {
    // expected
    assertTrue(true);
  }
}
 

import org.apache.avro.Schema; //导入方法依赖的package包/类
/**
 * Creates the avro record schema
 * 
 * @param recordName - Name of record.
 * @param columnNames - Column/field names
 * @param schemas - Avro schema for each field/column
 * @return - Avro Record schema
 */
public static Schema createAvroRecordSchema(String recordName, String[] columnNames,
    Schema[] schemas) {
  Schema record = Schema.createRecord(recordName, null, null, false);
  // create fields
  List<Schema.Field> fields = new ArrayList<>();
  // String field
  for (int i = 0; i < columnNames.length; i++) {
    String fieldName = columnNames[i];
    Schema fieldType = schemas[i];
    Schema.Field field = new Schema.Field(fieldName, fieldType, null, null,
        ParquetColumnConverterDescriptor.DEFAULT_ORDER);
    fields.add(field);
  }
  record.setFields(fields);
  return record;
}
 

import org.apache.avro.Schema; //导入方法依赖的package包/类
private Schema buildSchema(ColumnGenerator... extraCols) {
  List<Field> fields = new ArrayList<Field>();
  fields.add(buildField("id", Schema.Type.INT));
  fields.add(buildField("msg", Schema.Type.STRING));
  int colNum = 0;
  for (ColumnGenerator gen : extraCols) {
    if (gen.getColumnParquetSchema() != null) {
      fields.add(buildParquetField(forIdx(colNum++),
          gen.getColumnParquetSchema()));
    }
  }
  Schema schema = Schema.createRecord("myschema", null, null, false);
  schema.setFields(fields);
  return schema;
}
 

import org.apache.avro.Schema; //导入方法依赖的package包/类
@Test
public void testValidateSchemaPairMissingField() throws Exception {
  final List<Schema.Field> readerFields = list(
      new Schema.Field("oldfield1", INT_SCHEMA, null, null));
  final Schema reader = Schema.createRecord(readerFields);
  final SchemaCompatibility.SchemaPairCompatibility expectedResult =
      new SchemaCompatibility.SchemaPairCompatibility(
          SchemaCompatibility.SchemaCompatibilityResult.compatible(),
          reader,
          WRITER_SCHEMA,
          SchemaCompatibility.READER_WRITER_COMPATIBLE_MESSAGE);

  // Test omitting a field.
  assertEquals(expectedResult, checkReaderWriterCompatibility(reader, WRITER_SCHEMA));
}
 

import org.apache.avro.Schema; //导入方法依赖的package包/类
@Test
public void testValidateSchemaPairMissingSecondField() throws Exception {
  final List<Schema.Field> readerFields = list(
      new Schema.Field("oldfield2", STRING_SCHEMA, null, null));
  final Schema reader = Schema.createRecord(readerFields);
  final SchemaCompatibility.SchemaPairCompatibility expectedResult =
      new SchemaCompatibility.SchemaPairCompatibility(
          SchemaCompatibility.SchemaCompatibilityResult.compatible(),
          reader,
          WRITER_SCHEMA,
          SchemaCompatibility.READER_WRITER_COMPATIBLE_MESSAGE);

  // Test omitting other field.
  assertEquals(expectedResult, checkReaderWriterCompatibility(reader, WRITER_SCHEMA));
}
 

import org.apache.avro.Schema; //导入方法依赖的package包/类
public AvroRecordSchema( final StructContainerField schema ) throws IOException{
  this.schema = schema;

  List<Schema.Field> childList = new ArrayList<Schema.Field>();
  for( String key : schema.getKeys() ){
    childList.add( new Schema.Field( key , AvroSchemaFactory.getAvroSchema( schema.get( key ) ) , null , null ) );
  }
  avroSchema = Schema.createRecord( childList );
}
 

import org.apache.avro.Schema; //导入方法依赖的package包/类
public Schema generate(String schemaNameOverride) throws IOException {
  ClassWriter classWriter = new ClassWriter(options, connManager,
      tableName, null);
  Map<String, Integer> columnTypes = classWriter.getColumnTypes();
  String[] columnNames = classWriter.getColumnNames(columnTypes);

  List<Field> fields = new ArrayList<Field>();
  for (String columnName : columnNames) {
    String cleanedCol = AvroUtil.toAvroIdentifier(ClassWriter.toJavaIdentifier(columnName));
    int sqlType = columnTypes.get(columnName);
    Schema avroSchema = toAvroSchema(sqlType, columnName);
    Field field = new Field(cleanedCol, avroSchema, null,  NullNode.getInstance());
    field.addProp("columnName", columnName);
    field.addProp("sqlType", Integer.toString(sqlType));
    fields.add(field);
  }

  TableClassName tableClassName = new TableClassName(options);
  String shortClassName = tableClassName.getShortClassForTable(tableName);
  String avroTableName = (tableName == null ? TableClassName.QUERY_RESULT : tableName);
  String avroName = schemaNameOverride != null ? schemaNameOverride :
      (shortClassName == null ? avroTableName : shortClassName);
  String avroNamespace = tableClassName.getPackageForTable();

  String doc = "Sqoop import of " + avroTableName;
  Schema schema = Schema.createRecord(avroName, doc, avroNamespace, false);
  schema.setFields(fields);
  schema.addProp("tableName", avroTableName);
  return schema;
}
 

import org.apache.avro.Schema; //导入方法依赖的package包/类
private Schema buildAvroSchema(ColumnGenerator... extraCols) {
  List<Field> fields = new ArrayList<Field>();
  fields.add(buildAvroField("id", Schema.Type.INT));
  fields.add(buildAvroField("msg", Schema.Type.STRING));
  int colNum = 0;
  for (ColumnGenerator gen : extraCols) {
    if (gen.getColumnAvroSchema() != null) {
      fields.add(buildAvroField(forIdx(colNum++),
          gen.getColumnAvroSchema()));
    }
  }
  Schema schema = Schema.createRecord("myschema", null, null, false);
  schema.setFields(fields);
  return schema;
}
 

import org.apache.avro.Schema; //导入方法依赖的package包/类
private Schema computeSchema(String streamName, SamzaSqlRelMessage relMessage) {
  List<Schema.Field> keyFields = new ArrayList<>();
  List<String> fieldNames = relMessage.getFieldNames();
  List<Object> values = relMessage.getFieldValues();

  for (int index = 0; index < fieldNames.size(); index++) {
    if (fieldNames.get(index).equals(SamzaSqlRelMessage.KEY_NAME) || values.get(index) == null) {
      continue;
    }

    Object value = values.get(index);
    Schema avroType;
    if (value instanceof GenericData.Record) {
      avroType = ((GenericData.Record) value).getSchema();
    } else {
      avroType = ReflectData.get().getSchema(value.getClass());
    }
    keyFields.add(new Schema.Field(fieldNames.get(index), avroType, "", null));
  }

  Schema ks = Schema.createRecord(streamName, "", streamName + "_namespace", false);
  ks.setFields(keyFields);
  String schemaStr = ks.toString();
  Schema schema;
  // See whether we have a schema object corresponding to the schemaValue and reuse it.
  // CachedSchemaRegistryClient doesn't like if we recreate schema objects.
  if (_schemas.containsKey(schemaStr)) {
    schema = _schemas.get(schemaStr);
  } else {
    schema = Schema.parse(schemaStr);
    _schemas.put(schemaStr, schema);
  }

  return schema;
}
 

import org.apache.avro.Schema; //导入方法依赖的package包/类
private Schema createAvroSchema() {
  Schema avroSchema = Schema.createRecord("Doc", "adoc", null, false);
  List<Field> fields = new ArrayList<Field>();
  fields.add(new Field("id", Schema.create(Type.STRING), null, null));
  fields.add(new Field("user_friends_count",
                       createOptional(Schema.create(Type.INT)),
                       null, null));
  fields.add(new Field("user_location",
                       createOptional(Schema.create(Type.STRING)),
                       null, null));
  fields.add(new Field("user_description",
                       createOptional(Schema.create(Type.STRING)),
                       null, null));
  fields.add(new Field("user_statuses_count",
                       createOptional(Schema.create(Type.INT)),
                       null, null));
  fields.add(new Field("user_followers_count",
                       createOptional(Schema.create(Type.INT)),
                       null, null));
  fields.add(new Field("user_name",
                       createOptional(Schema.create(Type.STRING)),
                       null, null));
  fields.add(new Field("user_screen_name",
                       createOptional(Schema.create(Type.STRING)),
                       null, null));
  fields.add(new Field("created_at",
                       createOptional(Schema.create(Type.STRING)),
                       null, null));
  fields.add(new Field("text",
                       createOptional(Schema.create(Type.STRING)),
                       null, null));
  fields.add(new Field("retweet_count",
                       createOptional(Schema.create(Type.LONG)),
                       null, null));
  fields.add(new Field("retweeted",
                       createOptional(Schema.create(Type.BOOLEAN)),
                       null, null));
  fields.add(new Field("in_reply_to_user_id",
                       createOptional(Schema.create(Type.LONG)),
                       null, null));
  fields.add(new Field("source",
                       createOptional(Schema.create(Type.STRING)),
                       null, null));
  fields.add(new Field("in_reply_to_status_id",
                       createOptional(Schema.create(Type.LONG)),
                       null, null));
  fields.add(new Field("media_url_https",
                       createOptional(Schema.create(Type.STRING)),
                       null, null));
  fields.add(new Field("expanded_url",
                       createOptional(Schema.create(Type.STRING)),
                       null, null));
  avroSchema.setFields(fields);
  return avroSchema;
}
 

import org.apache.avro.Schema; //导入方法依赖的package包/类
public void createAvroFile(File file, String codec, boolean useSchemaUrl,
                           boolean useStaticSchemaUrl) throws IOException {
  // serialize a few events using the reflection-based avro serializer
  OutputStream out = new FileOutputStream(file);

  Context ctx = new Context();
  if (codec != null) {
    ctx.put("compressionCodec", codec);
  }

  Schema schema = Schema.createRecord("myrecord", null, null, false);
  schema.setFields(Arrays.asList(new Schema.Field[]{
      new Schema.Field("message", Schema.create(Schema.Type.STRING), null, null)
  }));
  GenericRecordBuilder recordBuilder = new GenericRecordBuilder(schema);
  File schemaFile = null;
  if (useSchemaUrl || useStaticSchemaUrl) {
    schemaFile = File.createTempFile(getClass().getSimpleName(), ".avsc");
    Files.write(schema.toString(), schemaFile, Charsets.UTF_8);
  }

  if (useStaticSchemaUrl) {
    ctx.put(AvroEventSerializerConfigurationConstants.STATIC_SCHEMA_URL,
            schemaFile.toURI().toURL().toExternalForm());
  }

  EventSerializer.Builder builder = new AvroEventSerializer.Builder();
  EventSerializer serializer = builder.build(ctx, out);

  serializer.afterCreate();
  for (int i = 0; i < 3; i++) {
    GenericRecord record = recordBuilder.set("message", "Hello " + i).build();
    Event event = EventBuilder.withBody(serializeAvro(record, schema));
    if (schemaFile == null && !useSchemaUrl) {
      event.getHeaders().put(AvroEventSerializer.AVRO_SCHEMA_LITERAL_HEADER,
          schema.toString());
    } else if (useSchemaUrl) {
      event.getHeaders().put(AvroEventSerializer.AVRO_SCHEMA_URL_HEADER,
          schemaFile.toURI().toURL().toExternalForm());
    }
    serializer.write(event);
  }
  serializer.flush();
  serializer.beforeClose();
  out.flush();
  out.close();
  if (schemaFile != null ) {
    schemaFile.delete();
  }

}