Java Schema.setFields方法代码示例

import org.apache.avro.Schema; //导入方法依赖的package包/类
public void testParquetRecordsNotSupported() throws IOException, SQLException {
  String[] argv = {};
  final int TOTAL_RECORDS = 1;

  Schema schema =  Schema.createRecord("nestedrecord", null, null, false);
  schema.setFields(Lists.newArrayList(buildField("myint",
      Schema.Type.INT)));
  GenericRecord record = new GenericData.Record(schema);
  record.put("myint", 100);
  // DB type is not used so can be anything:
  ColumnGenerator gen = colGenerator(record, schema, null, "VARCHAR(64)");
  createParquetFile(0, TOTAL_RECORDS,  gen);
  createTable(gen);
  try {
    runExport(getArgv(true, 10, 10, newStrArray(argv, "-m", "" + 1)));
    fail("Parquet records can not be exported.");
  } catch (Exception e) {
    // expected
    assertTrue(true);
  }
}
 

import org.apache.avro.Schema; //导入方法依赖的package包/类
public void testAvroRecordsNotSupported() throws IOException, SQLException {
  String[] argv = {};
  final int TOTAL_RECORDS = 1;

  Schema schema =  Schema.createRecord("nestedrecord", null, null, false);
  schema.setFields(Lists.newArrayList(buildAvroField("myint",
      Schema.Type.INT)));
  GenericRecord record = new GenericData.Record(schema);
  record.put("myint", 100);
  // DB type is not used so can be anything:
  ColumnGenerator gen = colGenerator(record, schema, null, "VARCHAR(64)");
  createAvroFile(0, TOTAL_RECORDS,  gen);
  createTable(gen);
  try {
    runExport(getArgv(true, 10, 10, newStrArray(argv, "-m", "" + 1)));
    fail("Avro records can not be exported.");
  } catch (Exception e) {
    // expected
    assertTrue(true);
  }
}
 

import org.apache.avro.Schema; //导入方法依赖的package包/类
/**
 * Creates the avro record schema
 * 
 * @param recordName - Name of record.
 * @param columnNames - Column/field names
 * @param schemas - Avro schema for each field/column
 * @return - Avro Record schema
 */
public static Schema createAvroRecordSchema(String recordName, String[] columnNames,
    Schema[] schemas) {
  Schema record = Schema.createRecord(recordName, null, null, false);
  // create fields
  List<Schema.Field> fields = new ArrayList<>();
  // String field
  for (int i = 0; i < columnNames.length; i++) {
    String fieldName = columnNames[i];
    Schema fieldType = schemas[i];
    Schema.Field field = new Schema.Field(fieldName, fieldType, null, null,
        ParquetColumnConverterDescriptor.DEFAULT_ORDER);
    fields.add(field);
  }
  record.setFields(fields);
  return record;
}
 

import org.apache.avro.Schema; //导入方法依赖的package包/类
public Schema generate(String schemaNameOverride) throws IOException {
  ClassWriter classWriter = new ClassWriter(options, connManager,
      tableName, null);
  Map<String, Integer> columnTypes = classWriter.getColumnTypes();
  String[] columnNames = classWriter.getColumnNames(columnTypes);

  List<Field> fields = new ArrayList<Field>();
  for (String columnName : columnNames) {
    String cleanedCol = AvroUtil.toAvroIdentifier(ClassWriter.toJavaIdentifier(columnName));
    int sqlType = columnTypes.get(columnName);
    Schema avroSchema = toAvroSchema(sqlType, columnName);
    Field field = new Field(cleanedCol, avroSchema, null,  NullNode.getInstance());
    field.addProp("columnName", columnName);
    field.addProp("sqlType", Integer.toString(sqlType));
    fields.add(field);
  }

  TableClassName tableClassName = new TableClassName(options);
  String shortClassName = tableClassName.getShortClassForTable(tableName);
  String avroTableName = (tableName == null ? TableClassName.QUERY_RESULT : tableName);
  String avroName = schemaNameOverride != null ? schemaNameOverride :
      (shortClassName == null ? avroTableName : shortClassName);
  String avroNamespace = tableClassName.getPackageForTable();

  String doc = "Sqoop import of " + avroTableName;
  Schema schema = Schema.createRecord(avroName, doc, avroNamespace, false);
  schema.setFields(fields);
  schema.addProp("tableName", avroTableName);
  return schema;
}
 

import org.apache.avro.Schema; //导入方法依赖的package包/类
private Schema buildSchema(ColumnGenerator... extraCols) {
  List<Field> fields = new ArrayList<Field>();
  fields.add(buildField("id", Schema.Type.INT));
  fields.add(buildField("msg", Schema.Type.STRING));
  int colNum = 0;
  for (ColumnGenerator gen : extraCols) {
    if (gen.getColumnParquetSchema() != null) {
      fields.add(buildParquetField(forIdx(colNum++),
          gen.getColumnParquetSchema()));
    }
  }
  Schema schema = Schema.createRecord("myschema", null, null, false);
  schema.setFields(fields);
  return schema;
}
 

import org.apache.avro.Schema; //导入方法依赖的package包/类
private Schema buildAvroSchema(ColumnGenerator... extraCols) {
  List<Field> fields = new ArrayList<Field>();
  fields.add(buildAvroField("id", Schema.Type.INT));
  fields.add(buildAvroField("msg", Schema.Type.STRING));
  int colNum = 0;
  for (ColumnGenerator gen : extraCols) {
    if (gen.getColumnAvroSchema() != null) {
      fields.add(buildAvroField(forIdx(colNum++),
          gen.getColumnAvroSchema()));
    }
  }
  Schema schema = Schema.createRecord("myschema", null, null, false);
  schema.setFields(fields);
  return schema;
}
 

import org.apache.avro.Schema; //导入方法依赖的package包/类
private Schema computeSchema(String streamName, SamzaSqlRelMessage relMessage) {
  List<Schema.Field> keyFields = new ArrayList<>();
  List<String> fieldNames = relMessage.getFieldNames();
  List<Object> values = relMessage.getFieldValues();

  for (int index = 0; index < fieldNames.size(); index++) {
    if (fieldNames.get(index).equals(SamzaSqlRelMessage.KEY_NAME) || values.get(index) == null) {
      continue;
    }

    Object value = values.get(index);
    Schema avroType;
    if (value instanceof GenericData.Record) {
      avroType = ((GenericData.Record) value).getSchema();
    } else {
      avroType = ReflectData.get().getSchema(value.getClass());
    }
    keyFields.add(new Schema.Field(fieldNames.get(index), avroType, "", null));
  }

  Schema ks = Schema.createRecord(streamName, "", streamName + "_namespace", false);
  ks.setFields(keyFields);
  String schemaStr = ks.toString();
  Schema schema;
  // See whether we have a schema object corresponding to the schemaValue and reuse it.
  // CachedSchemaRegistryClient doesn't like if we recreate schema objects.
  if (_schemas.containsKey(schemaStr)) {
    schema = _schemas.get(schemaStr);
  } else {
    schema = Schema.parse(schemaStr);
    _schemas.put(schemaStr, schema);
  }

  return schema;
}
 

import org.apache.avro.Schema; //导入方法依赖的package包/类
private Schema createAvroSchema() {
  Schema avroSchema = Schema.createRecord("Doc", "adoc", null, false);
  List<Field> fields = new ArrayList<Field>();
  fields.add(new Field("id", Schema.create(Type.STRING), null, null));
  fields.add(new Field("user_friends_count",
                       createOptional(Schema.create(Type.INT)),
                       null, null));
  fields.add(new Field("user_location",
                       createOptional(Schema.create(Type.STRING)),
                       null, null));
  fields.add(new Field("user_description",
                       createOptional(Schema.create(Type.STRING)),
                       null, null));
  fields.add(new Field("user_statuses_count",
                       createOptional(Schema.create(Type.INT)),
                       null, null));
  fields.add(new Field("user_followers_count",
                       createOptional(Schema.create(Type.INT)),
                       null, null));
  fields.add(new Field("user_name",
                       createOptional(Schema.create(Type.STRING)),
                       null, null));
  fields.add(new Field("user_screen_name",
                       createOptional(Schema.create(Type.STRING)),
                       null, null));
  fields.add(new Field("created_at",
                       createOptional(Schema.create(Type.STRING)),
                       null, null));
  fields.add(new Field("text",
                       createOptional(Schema.create(Type.STRING)),
                       null, null));
  fields.add(new Field("retweet_count",
                       createOptional(Schema.create(Type.LONG)),
                       null, null));
  fields.add(new Field("retweeted",
                       createOptional(Schema.create(Type.BOOLEAN)),
                       null, null));
  fields.add(new Field("in_reply_to_user_id",
                       createOptional(Schema.create(Type.LONG)),
                       null, null));
  fields.add(new Field("source",
                       createOptional(Schema.create(Type.STRING)),
                       null, null));
  fields.add(new Field("in_reply_to_status_id",
                       createOptional(Schema.create(Type.LONG)),
                       null, null));
  fields.add(new Field("media_url_https",
                       createOptional(Schema.create(Type.STRING)),
                       null, null));
  fields.add(new Field("expanded_url",
                       createOptional(Schema.create(Type.STRING)),
                       null, null));
  avroSchema.setFields(fields);
  return avroSchema;
}
 

import org.apache.avro.Schema; //导入方法依赖的package包/类
public void createAvroFile(File file, String codec, boolean useSchemaUrl,
                           boolean useStaticSchemaUrl) throws IOException {
  // serialize a few events using the reflection-based avro serializer
  OutputStream out = new FileOutputStream(file);

  Context ctx = new Context();
  if (codec != null) {
    ctx.put("compressionCodec", codec);
  }

  Schema schema = Schema.createRecord("myrecord", null, null, false);
  schema.setFields(Arrays.asList(new Schema.Field[]{
      new Schema.Field("message", Schema.create(Schema.Type.STRING), null, null)
  }));
  GenericRecordBuilder recordBuilder = new GenericRecordBuilder(schema);
  File schemaFile = null;
  if (useSchemaUrl || useStaticSchemaUrl) {
    schemaFile = File.createTempFile(getClass().getSimpleName(), ".avsc");
    Files.write(schema.toString(), schemaFile, Charsets.UTF_8);
  }

  if (useStaticSchemaUrl) {
    ctx.put(AvroEventSerializerConfigurationConstants.STATIC_SCHEMA_URL,
            schemaFile.toURI().toURL().toExternalForm());
  }

  EventSerializer.Builder builder = new AvroEventSerializer.Builder();
  EventSerializer serializer = builder.build(ctx, out);

  serializer.afterCreate();
  for (int i = 0; i < 3; i++) {
    GenericRecord record = recordBuilder.set("message", "Hello " + i).build();
    Event event = EventBuilder.withBody(serializeAvro(record, schema));
    if (schemaFile == null && !useSchemaUrl) {
      event.getHeaders().put(AvroEventSerializer.AVRO_SCHEMA_LITERAL_HEADER,
          schema.toString());
    } else if (useSchemaUrl) {
      event.getHeaders().put(AvroEventSerializer.AVRO_SCHEMA_URL_HEADER,
          schemaFile.toURI().toURL().toExternalForm());
    }
    serializer.write(event);
  }
  serializer.flush();
  serializer.beforeClose();
  out.flush();
  out.close();
  if (schemaFile != null ) {
    schemaFile.delete();
  }

}
 

import org.apache.avro.Schema; //导入方法依赖的package包/类
private Schema getAvroSchema() {
  List<Schema.Field> fields = new ArrayList<>();
  for (int i = 0; i < FILE_HEADER.length; i++) {

    Schema.Field field = new Schema.Field(FILE_HEADER[i],
        Schema.createUnion(
            Arrays.asList(Schema.create(Schema.Type.STRING), Schema.create(Schema.Type.NULL))),
        null, null);

    if (FILE_HEADER[i].equals("RowKey")) {
      field = new Schema.Field(FILE_HEADER[i],
          Schema.createUnion(
              Arrays.asList(Schema.create(Schema.Type.BYTES), Schema.create(Schema.Type.NULL))),
          null, null);
    }

    if (FILE_HEADER[i].equals("ID")) {
      field = new Schema.Field(FILE_HEADER[i],
          Schema.createUnion(
              Arrays.asList(Schema.create(Schema.Type.INT), Schema.create(Schema.Type.NULL))),
          null, null);
    }

    if (FILE_HEADER[i].equals("AGE")) {
      field = new Schema.Field(FILE_HEADER[i],
          Schema.createUnion(
              Arrays.asList(Schema.create(Schema.Type.INT), Schema.create(Schema.Type.NULL))),
          null, null);
    }

    if (FILE_HEADER[i].equals("SALARY")) {
      field = new Schema.Field(FILE_HEADER[i],
          Schema.createUnion(
              Arrays.asList(Schema.create(Schema.Type.DOUBLE), Schema.create(Schema.Type.NULL))),
          null, null);
    }

    if (FILE_HEADER[i].equals("VersionID")) {
      field = new Schema.Field(FILE_HEADER[i],
          Schema.createUnion(
              Arrays.asList(Schema.create(Schema.Type.LONG), Schema.create(Schema.Type.NULL))),
          null, null);
    }

    fields.add(field);
  }
  Schema avrorecord = Schema.createRecord("CDCRecord", null, null, false);
  avrorecord.setFields(fields);
  return avrorecord;
}