Java Schema类代码示例

import org.apache.avro.Schema; //导入依赖的package包/类
public static Schema complexUnion() {
  return SchemaBuilder
      .record("mine")
      .fields()
      .requiredLong("id")
      .name("str")
      .type(SchemaBuilder
          .unionOf()
          .stringType()
          .and()
          .longType()
          .and()
          .type(SchemaBuilder.record("inner").fields().requiredLong("a").requiredLong("b").endRecord())
          .and()
          .nullType()
          .endUnion())
      .noDefault()
      .endRecord();
}
 

import org.apache.avro.Schema; //导入依赖的package包/类
private String getIdFromRegistry(String subject, Schema schema) throws IOException {
  Map<String, Schema> idSchemaMap;
  if (idCache.containsKey(subject)) {
    idSchemaMap = idCache.get(subject);
    for (Map.Entry<String, Schema> entry: idSchemaMap.entrySet()) {
      if (entry.getValue().toString().equals(schema.toString())) {
        generateVersion(subject, schema);
        return entry.getKey();
      }
    }
  } else {
    idSchemaMap = new HashMap<String, Schema>();
  }
  String id = Integer.toString(ids.incrementAndGet());
  idSchemaMap.put(id, schema);
  idCache.put(subject, idSchemaMap);
  generateVersion(subject, schema);
  return id;
}
 

import org.apache.avro.Schema; //导入依赖的package包/类
@Override
public synchronized String register(String subject, Schema schema)
    throws IOException, RestClientException {
  Map<Schema, String> schemaIdMap;
  if (schemaCache.containsKey(subject)) {
    schemaIdMap = schemaCache.get(subject);
  } else {
    schemaIdMap = new IdentityHashMap<Schema, String>();
    schemaCache.put(subject, schemaIdMap);
  }

  if (schemaIdMap.containsKey(schema)) {
    return schemaIdMap.get(schema);
  } else {
    String id = getIdFromRegistry(subject, schema);
    schemaIdMap.put(schema, id);
    idCache.get(null).put(id, schema);
    return id;
  }
}
 

import org.apache.avro.Schema; //导入依赖的package包/类
private GenericRecord convertToAvroRecord(Schema avroRecordSchema, Object[] values) {
  // TODO can be improve to create once and reuse
  GenericRecord avroRec = new GenericData.Record(avroRecordSchema);
  List<ColumnConverterDescriptor> columnConverters = converterDescriptor.getColumnConverters();
  if (values.length != columnConverters.size()) {
    // mismatch schema
    // TODO better exception
    throw new RuntimeException("Expecting " + columnConverters.size() + " fields, received "
        + values.length + " values");
  }
  for (int i = 0; i < values.length; i++) {
    Object value = values[i];
    ColumnConverterDescriptor columnConverterDescriptor = columnConverters.get(i);
    Object valueToWrite = columnConverterDescriptor.getWritable(value);
    avroRec.put(columnConverterDescriptor.getColumnName(), valueToWrite);
  }
  return avroRec;
}
 

import org.apache.avro.Schema; //导入依赖的package包/类
public AvroFileInputStream(FileStatus status) throws IOException {
  pos = 0;
  buffer = new byte[0];
  GenericDatumReader<Object> reader = new GenericDatumReader<Object>();
  FileContext fc = FileContext.getFileContext(new Configuration());
  fileReader =
    DataFileReader.openReader(new AvroFSInput(fc, status.getPath()),reader);
  Schema schema = fileReader.getSchema();
  writer = new GenericDatumWriter<Object>(schema);
  output = new ByteArrayOutputStream();
  JsonGenerator generator =
    new JsonFactory().createJsonGenerator(output, JsonEncoding.UTF8);
  MinimalPrettyPrinter prettyPrinter = new MinimalPrettyPrinter();
  prettyPrinter.setRootValueSeparator(System.getProperty("line.separator"));
  generator.setPrettyPrinter(prettyPrinter);
  encoder = EncoderFactory.get().jsonEncoder(schema, generator);
}
 

import org.apache.avro.Schema; //导入依赖的package包/类
@Override
public synchronized int getVersion(String subject, Schema schema)
    throws IOException, RestClientException{
  Map<Schema, Integer> schemaVersionMap;
  if (versionCache.containsKey(subject)) {
    schemaVersionMap = versionCache.get(subject);
  } else {
    schemaVersionMap = new IdentityHashMap<Schema, Integer>();
    versionCache.put(subject, schemaVersionMap);
  }

  if (schemaVersionMap.containsKey(schema)) {
    return schemaVersionMap.get(schema);
  }  else {
    if (schemaVersionMap.size() >= identityMapCapacity) {
      throw new IllegalStateException("Too many schema objects created for " + subject + "!");
    }
    int version = getVersionFromRegistry(subject, schema);
    schemaVersionMap.put(schema, version);
    return version;
  }
}
 

import org.apache.avro.Schema; //导入依赖的package包/类
protected SqoopRecord toSqoopRecord(GenericRecord record) throws IOException {
  Schema avroSchema = record.getSchema();
  for (Map.Entry<Writable, Writable> e : columnTypes.entrySet()) {
    String columnName = e.getKey().toString();
    String columnType = e.getValue().toString();
    String cleanedCol = ClassWriter.toIdentifier(columnName);
    Schema.Field field = getFieldIgnoreCase(avroSchema, cleanedCol);
    if (null == field) {
      throw new IOException("Cannot find field " + cleanedCol
          + " in Avro schema " + avroSchema);
    }

    Object avroObject = record.get(field.name());
    Object fieldVal = AvroUtil.fromAvro(avroObject, field.schema(), columnType);
    recordImpl.setField(cleanedCol, fieldVal);
  }
  return recordImpl;
}
 

import org.apache.avro.Schema; //导入依赖的package包/类
@Test
public void publish_shouldThrowAnEncodeTransportExceptionForAnInvalidTransportContext() throws Exception {

	// expect
	exception.expect(EncodeTransportException.class);
	exception.expectCause(IsInstanceOf.<Throwable>instanceOf(NullPointerException.class));
	exception.expectMessage("Failed to publish message: Failed to encode transport");

	// given
	Context context = mock(Context.class);
	when(context.getSchema())
			.thenReturn(new Schema.Parser().parse("{\"name\":\"test\",\"type\":\"record\",\"fields\":[]}"));
	when(context.getDataBuffer()).thenReturn(null);

	GenericRecordBuilder builder = new GenericRecordBuilder(schema);
	builder.set("testString", "testString");
	Record record = builder.build();

	// when
	publisher.publish(context, record);

}
 

import org.apache.avro.Schema; //导入依赖的package包/类
public void testSpecifiedColumnsAsAvroFields()  throws IOException, SQLException {
  final int TOTAL_RECORDS = 10;
  ColumnGenerator[] gens = new ColumnGenerator[] {
    colGenerator(000, Schema.create(Schema.Type.INT), 100, "INTEGER"), //col0
    colGenerator(111, Schema.create(Schema.Type.INT), 100, "INTEGER"), //col1
    colGenerator(222, Schema.create(Schema.Type.INT), 100, "INTEGER"), //col2
    colGenerator(333, Schema.create(Schema.Type.INT), 100, "INTEGER")  //col3
  };
  createAvroFile(0, TOTAL_RECORDS, gens);
  createTable(gens);
  runExport(getArgv(true, 10, 10, newStrArray(null, "-m", "" + 1, "--columns", "id,msg,col1,col2")));
  verifyExport(TOTAL_RECORDS);
  assertColValForRowId(0, "col0", null);
  assertColValForRowId(0, "col1", 111);
  assertColValForRowId(0, "col2", 222);
  assertColValForRowId(0, "col3", null);
  assertColValForRowId(9, "col0", null);
  assertColValForRowId(9, "col1", 111);
  assertColValForRowId(9, "col2", 222);
  assertColValForRowId(9, "col3", null);
}
 

import org.apache.avro.Schema; //导入依赖的package包/类
@Override
public boolean isEnabled(NodeType type, String attributeName) {
	Schema fieldSchema = field.schema();
	Type schemaType = fieldSchema.getType();
	switch (attributeName) {
	case AvroAttributes.PRIMITIVE_TYPE:			
		if (PrimitiveType.isPrimitive(fieldSchema)) {
			return true;
		} else {				
			switch (schemaType) {
			case UNION:
				// it is enabled if there is only one child with primitive type
				return SchemaUtil.getSinglePrimitiveTypeOfUnion(fieldSchema) != null;
			case ARRAY:					
			case MAP:
				return PrimitiveType.getPrimitiveType(fieldSchema) != null;
			default:
				return false;
			}
		}
	case AvroAttributes.OPTIONAL:
		// optional is always enabled
		return true;
	}
	return true;
}
 

import org.apache.avro.Schema; //导入依赖的package包/类
public static Schema multiUnion() {
  return SchemaBuilder
      .record("mine")
      .fields()
      .requiredLong("id")
      .name("str")
      .type(SchemaBuilder.unionOf().longType().and().stringType().and().nullType().endUnion())
      .noDefault()
      .endRecord();
}
 

import org.apache.avro.Schema; //导入依赖的package包/类
@Test
public void matchArrayElementOfIncorrectType() throws Exception {
  Schema schema = SchemaBuilder.array().items().intType();
  JsonNode datum = mapper.readTree("[\"1\",\"2\"]");
  MatchType matchType = new JasvornoConverter(model, UndeclaredFieldBehaviour.NO_MATCH).matchArrayElement(datum,
      schema);
  assertThat(matchType, is(MatchType.NONE));
}
 

import org.apache.avro.Schema; //导入依赖的package包/类
public Schema generate(String schemaNameOverride) throws IOException {
  ClassWriter classWriter = new ClassWriter(options, connManager,
      tableName, null);
  Map<String, Integer> columnTypes = classWriter.getColumnTypes();
  String[] columnNames = classWriter.getColumnNames(columnTypes);

  List<Field> fields = new ArrayList<Field>();
  for (String columnName : columnNames) {
    String cleanedCol = AvroUtil.toAvroIdentifier(ClassWriter.toJavaIdentifier(columnName));
    int sqlType = columnTypes.get(columnName);
    Schema avroSchema = toAvroSchema(sqlType, columnName);
    Field field = new Field(cleanedCol, avroSchema, null,  NullNode.getInstance());
    field.addProp("columnName", columnName);
    field.addProp("sqlType", Integer.toString(sqlType));
    fields.add(field);
  }

  TableClassName tableClassName = new TableClassName(options);
  String shortClassName = tableClassName.getShortClassForTable(tableName);
  String avroTableName = (tableName == null ? TableClassName.QUERY_RESULT : tableName);
  String avroName = schemaNameOverride != null ? schemaNameOverride :
      (shortClassName == null ? avroTableName : shortClassName);
  String avroNamespace = tableClassName.getPackageForTable();

  String doc = "Sqoop import of " + avroTableName;
  Schema schema = Schema.createRecord(avroName, doc, avroNamespace, false);
  schema.setFields(fields);
  schema.addProp("tableName", avroTableName);
  return schema;
}
 

import org.apache.avro.Schema; //导入依赖的package包/类
private SchemaCompatibilityResult checkFixedSize(final Schema reader, final Schema writer, final Stack<String> location) {
  SchemaCompatibilityResult result = SchemaCompatibilityResult.compatible();
  location.push("size");
  int actual = reader.getFixedSize();
  int expected = writer.getFixedSize();
  if (actual != expected) {
    String message = String.format("expected: %d, found: %d", expected, actual);
    result = SchemaCompatibilityResult.incompatible(
        SchemaIncompatibilityType.FIXED_SIZE_MISMATCH, reader,
        writer, message, location);
  }
  // POP "size" literal
  location.pop();
  return result;
}
 

import org.apache.avro.Schema; //导入依赖的package包/类
@Override
public String[] getAllKey() throws IOException{
  Schema schema = record.getSchema();
  List<Schema.Field> childFieldList = schema.getFields();
  String[] keys = new String[childFieldList.size()];
  for( int i = 0 ; i < childFieldList.size() ; i++ ){
    keys[i] = childFieldList.get(i).name();
  }

  return keys;
}