本文整理汇总了Scala中org.apache.avro.SchemaBuilder类的典型用法代码示例。如果您正苦于以下问题：Scala SchemaBuilder类的具体用法？Scala SchemaBuilder怎么用？Scala SchemaBuilder使用的例子？那么, 这里精选的类代码示例或许可以为您提供帮助。
在下文中一共展示了SchemaBuilder类的4个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Scala代码示例。
示例1: AvroUtils
//设置package包名称以及导入依赖的类
package com.datawizards.sparklocal.impl.scala.dataset.io
import org.apache.avro.generic.GenericRecord
import org.apache.avro.{Schema, SchemaBuilder}
import scala.collection.JavaConversions._
object AvroUtils {
def mapSchema(originalSchema: Schema, fieldNameMapping: Map[String, String]): Schema = {
var customSchemaBuilder = SchemaBuilder
.builder(originalSchema.getNamespace)
.record(originalSchema.getName)
.fields()
originalSchema.getFields.toList.foreach { f =>
customSchemaBuilder = customSchemaBuilder
.name(fieldNameMapping(f.name()))
.`type`(f.schema())
.noDefault()
}
customSchemaBuilder.endRecord()
}
def mapGenericRecordFromOriginalToTarget(record: GenericRecord, mappedSchema: Schema, fieldNameMapping: Map[String, String]): GenericRecord = {
val customRecord = new org.apache.avro.generic.GenericData.Record(mappedSchema)
record.getSchema.getFields.foreach{ f =>
customRecord.put(fieldNameMapping(f.name()), record.get(f.name()))
}
customRecord
}
def mapGenericRecordFromTargetToOriginal(record: GenericRecord, schema: Schema, fieldNameMapping: Map[String, String]): GenericRecord = {
val customRecord = new org.apache.avro.generic.GenericData.Record(schema)
customRecord.getSchema.getFields.foreach{ f =>
customRecord.put(f.name(), record.get(fieldNameMapping(f.name())))
}
customRecord
}
} 示例2: AvroSchemaFnsTest
//设置package包名称以及导入依赖的类
package io.eels.component.avro
import java.util
import io.eels.schema._
import org.apache.avro.SchemaBuilder
import org.codehaus.jackson.node.NullNode
import org.scalatest.{Matchers, WordSpec}
import scala.collection.JavaConverters._
class AvroSchemaFnsTest extends WordSpec with Matchers {
"toAvro" should {
"use a union of [null, type] for a nullable column" in {
val schema = StructType(Field("a", StringType, true))
val fields = AvroSchemaFns.toAvroSchema(schema).getFields.asScala
fields.head.schema().getType shouldBe org.apache.avro.Schema.Type.UNION
fields.head.schema().getTypes.get(0).getType shouldBe org.apache.avro.Schema.Type.NULL
fields.head.schema().getTypes.get(1).getType shouldBe org.apache.avro.Schema.Type.STRING
}
"set default type of NullNode for a nullable column" in {
val schema = StructType(Field("a", StringType, true))
val fields = AvroSchemaFns.toAvroSchema(schema).getFields
fields.get(0).defaultValue() shouldBe NullNode.getInstance()
}
"not set a default value for a non null column" in {
val schema = StructType(Field("a", IntType(true), false))
val fields = AvroSchemaFns.toAvroSchema(schema).getFields
(fields.get(0).defaultVal() == null) shouldBe true
fields.get(0).schema().getType shouldBe org.apache.avro.Schema.Type.INT
}
}
"fromAvroSchema" should {
"convert avro unions [null, string] to nullable columns" in {
val avro = SchemaBuilder.record("dummy").fields().optionalString("str").endRecord()
AvroSchemaFns.fromAvroSchema(avro) shouldBe StructType(Field("str", StringType, true))
}
"convert avro unions [null, double] to nullable double columns" in {
val union = org.apache.avro.Schema.createUnion(util.Arrays.asList(SchemaBuilder.builder().doubleType(), SchemaBuilder.builder().nullType()))
val avro = SchemaBuilder.record("dummy").fields().name("u").`type`(union).noDefault().endRecord()
AvroSchemaFns.fromAvroSchema(avro) shouldBe StructType(Field("u", DoubleType, true))
}
}
} 示例3: AvroSchemaMergeTest
//设置package包名称以及导入依赖的类
package io.eels.component.avro
import org.apache.avro.SchemaBuilder
import org.scalatest.{Matchers, WordSpec}
class AvroSchemaMergeTest extends WordSpec with Matchers {
"AvroSchemaMerge" should {
"merge all fields" in {
val schema1 = SchemaBuilder.record("record1").fields().nullableString("str1", "moo").requiredFloat("f").endRecord()
val schema2 = SchemaBuilder.record("record2").fields().nullableString("str2", "foo").requiredFloat("g").endRecord()
AvroSchemaMerge("finalname", "finalnamespace", List(schema1, schema2)) shouldBe
SchemaBuilder.record("finalname").namespace("finalnamespace")
.fields()
.nullableString("str1", "moo")
.requiredFloat("f")
.nullableString("str2", "foo")
.requiredFloat("g")
.endRecord()
}
"drop duplicates" in {
val schema1 = SchemaBuilder.record("record1").fields().nullableString("str1", "moo").requiredFloat("f").endRecord()
val schema2 = SchemaBuilder.record("record2").fields().nullableString("str2", "foo").requiredFloat("f").endRecord()
AvroSchemaMerge("finalname", "finalnamespace", List(schema1, schema2)) shouldBe
SchemaBuilder.record("finalname").namespace("finalnamespace")
.fields()
.nullableString("str1", "moo")
.requiredFloat("f")
.nullableString("str2", "foo")
.endRecord()
}
}
} 示例4: ConverterTest
//设置package包名称以及导入依赖的类
package io.eels.component.avro
import org.apache.avro.SchemaBuilder
import org.scalatest.{Matchers, WordSpec}
class ConverterTest extends WordSpec with Matchers {
"Converter" should {
"convert to long" in {
AvroSerializer(SchemaBuilder.builder().longType()).serialize("123") shouldBe 123l
AvroSerializer(SchemaBuilder.builder().longType()).serialize(14555) shouldBe 14555l
}
"convert to String" in {
AvroSerializer(SchemaBuilder.builder().stringType()).serialize(123l) shouldBe "123"
AvroSerializer(SchemaBuilder.builder().stringType).serialize(124) shouldBe "124"
AvroSerializer(SchemaBuilder.builder().stringType).serialize("Qweqwe") shouldBe "Qweqwe"
}
"convert to boolean" in {
AvroSerializer(SchemaBuilder.builder().booleanType).serialize(true) shouldBe true
AvroSerializer(SchemaBuilder.builder().booleanType).serialize(false) shouldBe false
AvroSerializer(SchemaBuilder.builder().booleanType).serialize("true") shouldBe true
AvroSerializer(SchemaBuilder.builder().booleanType()).serialize("false") shouldBe false
}
"convert to Double" in {
AvroSerializer(SchemaBuilder.builder().doubleType).serialize("213.4") shouldBe 213.4d
AvroSerializer(SchemaBuilder.builder().doubleType).serialize("345.11") shouldBe 345.11d
AvroSerializer(SchemaBuilder.builder().doubleType()).serialize(345) shouldBe 345.0
}
}
}