本文整理汇总了Scala中org.apache.avro.io.DecoderFactory类的典型用法代码示例。如果您正苦于以下问题:Scala DecoderFactory类的具体用法?Scala DecoderFactory怎么用?Scala DecoderFactory使用的例子?那么恭喜您, 这里精选的类代码示例或许可以为您提供帮助。
在下文中一共展示了DecoderFactory类的7个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Scala代码示例。
示例1: Tip
//设置package包名称以及导入依赖的类
package com.alvin.niagara.model
import java.io.ByteArrayOutputStream
import java.util
import org.apache.avro.Schema
import org.apache.avro.generic.{GenericData, GenericDatumReader, GenericDatumWriter, GenericRecord}
import org.apache.avro.io.{DecoderFactory, EncoderFactory}
import scala.collection.JavaConversions._
import scala.io.Source
case class Tip(business_id: String, date: String, likes: Long, text: String, `type`: String, user_id: String)
object TipSerde {
val avroSchema = Source.fromInputStream(getClass.getResourceAsStream("/schema/tip.avsc")).mkString
val schema = new Schema.Parser().parse(avroSchema)
val reader = new GenericDatumReader[GenericRecord](schema)
val writer = new GenericDatumWriter[GenericRecord](schema)
def serialize(tip: Tip): Array[Byte] = {
val out = new ByteArrayOutputStream()
val encoder = EncoderFactory.get.binaryEncoder(out, null)
val avroRecord = new GenericData.Record(schema)
avroRecord.put("business_id", tip.business_id)
avroRecord.put("date", tip.date)
avroRecord.put("likes", tip.likes)
avroRecord.put("text", tip.text)
avroRecord.put("type", tip.`type`)
avroRecord.put("user_id", tip.user_id)
writer.write(avroRecord, encoder)
encoder.flush
out.close
out.toByteArray
}
def deserialize(bytes: Array[Byte]): Tip = {
val decoder = DecoderFactory.get.binaryDecoder(bytes, null)
val record = reader.read(null, decoder)
Tip(
record.get("business_id").toString,
record.get("date").toString,
record.get("likes").asInstanceOf[Long],
record.get("text").toString,
record.get("type").toString,
record.get("user_id").toString
)
}
}
示例2: Avro
//设置package包名称以及导入依赖的类
package com.lukecycon.avro
import java.io.ByteArrayOutputStream
import org.apache.avro.io.EncoderFactory
import org.apache.avro.file.BZip2Codec
import java.nio.ByteBuffer
import org.apache.avro.io.DecoderFactory
object Avro {
def schemaFor[T: AvroFormat] = implicitly[AvroFormat[T]].schema
def write[T: AvroFormat](thing: T, compress: Boolean = false): Array[Byte] = {
val out = new ByteArrayOutputStream
val encoder = EncoderFactory.get.binaryEncoder(out, null)
implicitly[AvroFormat[T]].writeValue(thing, encoder)
encoder.flush
if (compress) {
new BZip2Codec().compress(ByteBuffer.wrap(out.toByteArray)).array
} else {
out.toByteArray
}
}
def writeHex[T: AvroFormat](thing: T): String =
byteArrayToHexString(write(thing))
def read[T: AvroFormat](bytes: Array[Byte],
compressed: Boolean = false): Either[String, T] = {
val byts = if (compressed) {
new BZip2Codec().decompress(ByteBuffer.wrap(bytes)).array
} else {
bytes
}
val decoder = DecoderFactory.get.binaryDecoder(byts, null)
implicitly[AvroFormat[T]].decodeValue(Nil, decoder)
}
def readHex[T: AvroFormat](hex: String): Either[String, T] =
read(
hex
.replace(" ", "")
.grouped(2)
.map(Integer.parseInt(_, 16).toByte)
.toArray)
private def byteArrayToHexString(bb: Array[Byte]): String =
bb.map("%02X" format _).mkString.grouped(2).mkString(" ")
}
示例3: AvroUtils
//设置package包名称以及导入依赖的类
package pulse.kafka.avro
import java.io.{ByteArrayInputStream, ByteArrayOutputStream, DataInputStream, File}
import com.twitter.util.Future
import org.apache.avro.Schema
import org.apache.avro.file.DataFileWriter
import org.apache.avro.generic.{GenericDatumReader, GenericDatumWriter, GenericRecord}
import org.apache.avro.io.DecoderFactory
import pulse.kafka.extensions.managedByteArrayInputStream
import pulse.kafka.extensions.managedByteArrayOutputStream
import pulse.kafka.extensions.catsStdInstancesForFuture
import scala.concurrent.ExecutionContext.Implicits._
object AvroUtils {
import pulse.common.syntax._
def jsonToAvroBytes(json: String, schemaFile: File): Future[Array[Byte]] =
use(new ByteArrayOutputStream()) { output =>
for {
s <- loadSchema(schemaFile)
_ <- convertImpl(json, output, s)
} yield output.toByteArray
}
private def convertImpl(json: String, output: ByteArrayOutputStream, schemaSpec: Schema): Future[GenericDatumReader[GenericRecord]] =
use(new ByteArrayInputStream(json.getBytes)) { input =>
for {
w <- getWriter(output, schemaSpec)
r <- getReader(input, schemaSpec, w)
} yield r
}
def getReader(input: ByteArrayInputStream, schemaSpec: Schema, w: DataFileWriter[GenericRecord]) = Future.value {
val reader = new GenericDatumReader[GenericRecord](schemaSpec)
val datum = reader.read(null, getJsonDecoder(input, schemaSpec))
w.append(datum)
w.flush()
reader
}
private def getJsonDecoder(input: ByteArrayInputStream, schema: Schema) =
DecoderFactory.get.jsonDecoder(schema, new DataInputStream(input))
private def getWriter(output: ByteArrayOutputStream, schemaSpec: Schema) = {
Future.value {
val writer = new DataFileWriter[GenericRecord](new GenericDatumWriter[GenericRecord]())
writer.create(schemaSpec, output)
}
}
private def loadSchema(schemaFile: File): Future[Schema] =
Future {
new Schema.Parser().parse(schemaFile)
}
}
示例4: AvroUtils
//设置package包名称以及导入依赖的类
package pulse.services.example.avro
import java.io.{ByteArrayInputStream, ByteArrayOutputStream, DataInputStream, File}
import org.apache.avro.Schema
import org.apache.avro.file.DataFileWriter
import org.apache.avro.generic.{GenericDatumReader, GenericDatumWriter, GenericRecord}
import org.apache.avro.io.DecoderFactory
import pulse.services.example.extensions._
object AvroUtils {
def jsonToAvroBytes(json: String, schemaFile: File) = {
use(new ByteArrayOutputStream())(output => {
val schemaSpec = loadSchema(schemaFile)
use(new ByteArrayInputStream(json.getBytes))(input => {
val writer = new DataFileWriter[GenericRecord](new GenericDatumWriter[GenericRecord]())
writer.create(schemaSpec, output)
val reader = new GenericDatumReader[GenericRecord](schemaSpec)
val datum = reader.read(null, getJsonDecoder(input, schemaSpec))
writer.append(datum)
writer.flush()
})
output.toByteArray
})
}
def getJsonDecoder(input: ByteArrayInputStream, schema: Schema) =
DecoderFactory.get.jsonDecoder(schema, new DataInputStream(input))
def loadSchema(schemaFile: File) =
new Schema.Parser().parse(schemaFile)
}
示例5: AvroFlumeEventDecoder
//设置package包名称以及导入依赖的类
package contrib.kafka.serializer
import kafka.serializer.Decoder
import kafka.utils.VerifiableProperties
import org.apache.avro.io.BinaryDecoder
import org.apache.avro.io.DecoderFactory
import org.apache.avro.specific.SpecificDatumReader
import org.apache.flume.Event
import org.apache.flume.event.EventBuilder
import org.apache.flume.source.avro.AvroFlumeEvent
import java.io.ByteArrayInputStream
class AvroFlumeEventDecoder(props: VerifiableProperties = null)
extends Decoder[Event] {
private val reader: SpecificDatumReader[AvroFlumeEvent] =
new SpecificDatumReader[AvroFlumeEvent](classOf[AvroFlumeEvent])
private var decoder: BinaryDecoder = null.asInstanceOf[BinaryDecoder]
override def fromBytes(bytes: Array[Byte]): Event = {
val inputStream = new ByteArrayInputStream(bytes)
decoder = DecoderFactory.get.directBinaryDecoder(inputStream, decoder)
val avroEvent: AvroFlumeEvent = reader.read(null, decoder)
EventBuilder.withBody(
avroEvent.getBody.array,
toStringJavaMap(avroEvent.getHeaders))
}
def toStringJavaMap(
charSeqMap: JMap[CharSequence, CharSequence]): JMap[String, String] = {
import scala.collection.JavaConversions._
for ((k: CharSequence, v: CharSequence) <- charSeqMap)
yield (k.toString, v.toString)
}
}
示例6: mkSerializer
//设置package包名称以及导入依赖的类
package com.cj.serialization
import java.io.ByteArrayOutputStream
import org.apache.avro.Schema
import org.apache.avro.io.{BinaryDecoder, BinaryEncoder, DatumWriter, DecoderFactory, EncoderFactory}
import org.apache.avro.specific.{SpecificDatumReader, SpecificDatumWriter, SpecificRecord}
package object avro {
type RecordSerializer[-T] = T => Array[Byte]
type RecordDeserializer[+T] = Array[Byte] => T
// make a RecordSerializer for class T, passing through intermediate avro-generated class U
def mkSerializer[T, U <: SpecificRecord](f: (T => U)): RecordSerializer[T] = {
val avroSerializer = mkAvroSerializer[U]()
record => avroSerializer(f(record))
}
// make a RecordDeserializer for class T, passing through intermediate avro-generated class U
def mkDeserializer[T, U >: Null <: SpecificRecord](f: U => T, schema: Schema): RecordDeserializer[Option[T]] = {
val avroDeserializer = mkAvroDeserializer(schema)
bytes => {
val avroRec: U = avroDeserializer(bytes)
if (avroRec == null) None
else Some(f(avroRec))
}
}
// make a RecordSerializer for avro-generated class T
def mkAvroSerializer[T <: SpecificRecord](): RecordSerializer[T] = {
val output = new ByteArrayOutputStream()
val writer: DatumWriter[T] = new SpecificDatumWriter[T]()
var encoder: BinaryEncoder = EncoderFactory.get().binaryEncoder(output, null)
record => {
output.reset()
encoder = EncoderFactory.get().binaryEncoder(output, encoder)
writer.setSchema(record.getSchema)
writer.write(record, encoder)
encoder.flush()
output.close()
output.toByteArray
}
}
// make a RecordDeserializer for avro-generated class T
def mkAvroDeserializer[T >: Null <: SpecificRecord](schema: Schema): RecordDeserializer[T] = {
val reader: SpecificDatumReader[T] = new SpecificDatumReader[T](schema)
var decoder: BinaryDecoder = DecoderFactory.get().binaryDecoder(Array[Byte](), null)
bytes => {
decoder = DecoderFactory.get().binaryDecoder(bytes, decoder)
reader.read(null, decoder)
}
}
}
示例7: JsonEncodingDecoding
//设置package包名称以及导入依赖的类
package com.giampaolotrapasso.avrosamples.test
import java.io.ByteArrayInputStream
import com.giampaolotrapasso.avrosamples.SchemaRegistry
import com.giampaolotrapasso.avrosamples.events.{Event, MovieChangedV1, MovieChangedV2, MovieChangedV3}
import com.giampaolotrapasso.avrosamples.serializers.{BinarySerializer, DataWithSchemaSerializer, JsonSerializer}
import com.sksamuel.avro4s.{AvroInputStream, FromRecord, RecordFormat, ToRecord}
import org.apache.avro.Schema
import org.apache.avro.file.SeekableByteArrayInput
import org.apache.avro.generic.{GenericDatumReader, GenericRecord}
import org.apache.avro.io.DecoderFactory
class JsonEncodingDecoding extends TestSpec {
val title = "Raiders of lost ark"
val year = 1986
val director = "Spielberg"
val wonOscars = 1
def deserialize[A <: Event: ToRecord: FromRecord: RecordFormat](oldSchema: Schema,
newSchema: Schema,
stream: ByteArrayInputStream) = {
val gdr = new GenericDatumReader[GenericRecord](oldSchema, newSchema)
val binDecoder = DecoderFactory.get().jsonDecoder(newSchema, stream)
val record: GenericRecord = gdr.read(null, binDecoder)
val format = RecordFormat[A]
format.from(record)
}
"BinaryEncodingDecodingTest" should "deserialize an added field V1(title, year) to V2(title, year, director) " in {
val obj = MovieChangedV1(title, year)
val bytes: Array[Byte] = JsonSerializer.serializeV1(obj)
//println("*** Json Size" + bytes.length)
val in = new SeekableByteArrayInput(bytes)
val result = deserialize[MovieChangedV2](SchemaRegistry.movieChanged(1), SchemaRegistry.movieChanged(2), in)
result should matchPattern {
case MovieChangedV2(`title`, `year`, "unknown") ?
}
}
}