本文整理汇总了Scala中akka.kafka.ConsumerSettings类的典型用法代码示例。如果您正苦于以下问题:Scala ConsumerSettings类的具体用法?Scala ConsumerSettings怎么用?Scala ConsumerSettings使用的例子?那么, 这里精选的类代码示例或许可以为您提供帮助。
在下文中一共展示了ConsumerSettings类的15个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Scala代码示例。
示例1: ProcessingKafkaApplication
//设置package包名称以及导入依赖的类
package com.packt.chapter8
import akka.actor.ActorSystem
import akka.kafka.scaladsl.{Consumer, Producer}
import akka.kafka.{ConsumerSettings, ProducerSettings, Subscriptions}
import akka.stream.{ActorMaterializer, ClosedShape}
import akka.stream.scaladsl.{Flow, GraphDSL, RunnableGraph, Sink, Source}
import org.apache.kafka.clients.consumer.{ConsumerConfig, ConsumerRecord}
import org.apache.kafka.clients.producer.ProducerRecord
import org.apache.kafka.common.TopicPartition
import org.apache.kafka.common.serialization.{ByteArrayDeserializer, ByteArraySerializer, StringDeserializer, StringSerializer}
import scala.concurrent.duration._
object ProcessingKafkaApplication extends App {
implicit val actorSystem = ActorSystem("SimpleStream")
implicit val actorMaterializer = ActorMaterializer()
val bootstrapServers = "localhost:9092"
val kafkaTopic = "akka_streams_topic"
val partition = 0
val subscription = Subscriptions.assignment(new TopicPartition(kafkaTopic, partition))
val consumerSettings = ConsumerSettings(actorSystem, new ByteArrayDeserializer, new StringDeserializer)
.withBootstrapServers(bootstrapServers)
.withGroupId("akka_streams_group")
.withProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest")
val producerSettings = ProducerSettings(actorSystem, new ByteArraySerializer, new StringSerializer)
.withBootstrapServers(bootstrapServers)
val runnableGraph = RunnableGraph.fromGraph(GraphDSL.create() { implicit builder =>
import GraphDSL.Implicits._
val tickSource = Source.tick(0 seconds, 5 seconds, "Hello from Akka Streams using Kafka!")
val kafkaSource = Consumer.plainSource(consumerSettings, subscription)
val kafkaSink = Producer.plainSink(producerSettings)
val printlnSink = Sink.foreach(println)
val mapToProducerRecord = Flow[String].map(elem => new ProducerRecord[Array[Byte], String](kafkaTopic, elem))
val mapFromConsumerRecord = Flow[ConsumerRecord[Array[Byte], String]].map(record => record.value())
tickSource ~> mapToProducerRecord ~> kafkaSink
kafkaSource ~> mapFromConsumerRecord ~> printlnSink
ClosedShape
})
runnableGraph.run()
}
示例2: Pusher
//设置package包名称以及导入依赖的类
package reactivehub.akka.stream.apns.pusher
import akka.actor.ActorSystem
import akka.kafka.ConsumerSettings
import akka.kafka.scaladsl.Consumer
import akka.stream.ActorMaterializer
import akka.stream.scaladsl.{Keep, Sink}
import io.netty.channel.nio.NioEventLoopGroup
import io.netty.handler.ssl.SslContext
import org.apache.kafka.clients.consumer.ConsumerConfig.AUTO_OFFSET_RESET_CONFIG
import reactivehub.akka.stream.apns.Environment._
import reactivehub.akka.stream.apns.TlsUtil._
import reactivehub.akka.stream.apns._
import reactivehub.akka.stream.apns.marshallers.SprayJsonSupport
object Pusher extends SprayJsonSupport {
val kafka = "192.168.99.100:9092"
val clientId = "pusher1"
val consumerGroup = "pusher"
val topics = Set("notifications")
implicit val system = ActorSystem("system")
implicit val materializer = ActorMaterializer()
import system.dispatcher
def main(args: Array[String]): Unit = {
val group = new NioEventLoopGroup()
val apns = ApnsExt(system).connection[Long](Development, sslContext, group)
Consumer.atMostOnceSource(consumerSettings)
.map(msg => msg.key -> toNotification(msg.value))
.filter(_._2.deviceToken.bytes.length < 100)
.viaMat(apns)(Keep.right)
.log("pusher", _.toString())
.to(Sink.ignore).run()
.onComplete { _ =>
group.shutdownGracefully()
system.terminate()
}
}
private def sslContext: SslContext =
loadPkcs12FromResource("/cert.p12", "password")
private def consumerSettings: ConsumerSettings[Long, PushData] =
ConsumerSettings(system, ScalaLongDeserializer, PushDataDeserializer, topics)
.withBootstrapServers(kafka)
.withClientId(clientId)
.withGroupId(consumerGroup)
.withProperty(AUTO_OFFSET_RESET_CONFIG, "earliest")
private def toNotification(pushData: PushData): Notification = {
var builder = Payload.Builder()
pushData.alert.foreach(alert => builder = builder.withAlert(alert))
pushData.badge.foreach(badge => builder = builder.withBadge(badge))
Notification(DeviceToken(pushData.token), builder.result)
}
}
示例3: Settings
//设置package包名称以及导入依赖的类
package com.scalaio.kafka.consumer
import akka.actor.ActorSystem
import akka.kafka.ConsumerMessage.CommittableMessage
import akka.kafka.scaladsl.Consumer
import akka.kafka.{ConsumerSettings, ProducerSettings, Subscriptions}
import akka.stream.ActorMaterializer
import akka.stream.scaladsl.Sink
import com.scalaio.kafka.consumer.Settings.consumerSettings
import org.apache.kafka.clients.consumer.ConsumerConfig
import org.apache.kafka.common.serialization.{ByteArrayDeserializer, ByteArraySerializer, StringDeserializer, StringSerializer}
import scala.concurrent.Future
object Settings {
def consumerSettings(implicit system: ActorSystem) =
ConsumerSettings(system, new ByteArrayDeserializer, new StringDeserializer)
.withBootstrapServers("localhost:9092")
.withGroupId("CommittableSourceConsumer")
.withProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest")
def producerSettings(implicit system: ActorSystem) =
ProducerSettings(system, new ByteArraySerializer, new StringSerializer)
.withBootstrapServers("localhost:9092")
}
object CommittableSource extends App {
type KafkaMessage = CommittableMessage[Array[Byte], String]
implicit val system = ActorSystem("CommittableSourceConsumerMain")
implicit val materializer = ActorMaterializer()
implicit val ec = system.dispatcher
// explicit commit
Consumer
.committableSource(consumerSettings, Subscriptions.topics("topic1"))
.mapAsync(1) { msg =>
BusinessController.handleMessage(msg.record.value)
.flatMap(response => msg.committableOffset.commitScaladsl())
.recoverWith { case e => msg.committableOffset.commitScaladsl() }
}
.runWith(Sink.ignore)
}
object BusinessController {
type Service[A, B] = A => Future[B]
val handleMessage: Service[String, String] =
(message) => Future.successful(message.toUpperCase)
}
示例4: PacketConsumer
//设置package包名称以及导入依赖的类
package edu.uw.at.iroberts.wirefugue.kafka.consumer
import akka.actor.ActorSystem
import akka.kafka.scaladsl.Consumer
import akka.kafka.{ConsumerSettings, Subscriptions}
import akka.stream.ActorMaterializer
import akka.stream.scaladsl.{Keep, Sink, Source}
import com.typesafe.config.ConfigFactory
import edu.uw.at.iroberts.wirefugue.kafka.producer.KafkaKey
import edu.uw.at.iroberts.wirefugue.kafka.serdes.{PacketDeserializer, PacketSerde}
import edu.uw.at.iroberts.wirefugue.pcap.Packet
import org.apache.kafka.clients.consumer.{ConsumerConfig, ConsumerRecord}
import org.apache.kafka.common.serialization.IntegerDeserializer
import scala.concurrent.Await
import scala.concurrent.duration._
object PacketConsumer extends App {
type PacketRecord = ConsumerRecord[KafkaKey, Array[Byte]]
val config = ConfigFactory.load("application.conf")
implicit val system = ActorSystem("stream-consumer-system", config)
implicit val materializer = ActorMaterializer()
val consumerSettings = ConsumerSettings(system, new IntegerDeserializer, new PacketDeserializer)
.withGroupId("group1")
.withProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "latest")
// Separate streams for each partition
val maxPartitions = 100
val consumerGroup = Consumer.plainPartitionedSource(consumerSettings, Subscriptions.topics("packets"))
val done = consumerGroup.map {
case (topicPartition, source) =>
val p: Int = topicPartition.partition
source
.map { (cr: ConsumerRecord[Integer, Packet]) => cr.value() }
.filter(_.ip.isDefined)
.toMat(Sink.foreach(packet => println(s"[$p] $packet")))(Keep.both)
.run()
}
.mapAsyncUnordered(maxPartitions)(_._2)
.runWith(Sink.ignore)
Await.result(done, Duration.Inf)
system.terminate()
}
示例5: StreamConsumer
//设置package包名称以及导入依赖的类
package consumers
import akka.Done
import akka.actor.ActorSystem
import akka.kafka.scaladsl.Consumer
import akka.kafka.{ConsumerSettings, Subscriptions}
import akka.stream.scaladsl.Sink
import akka.stream.{ActorMaterializer, ActorMaterializerSettings}
import cats.data.Xor
import com.typesafe.config.ConfigFactory
import org.apache.kafka.clients.consumer.{ConsumerConfig, ConsumerRecord}
import org.apache.kafka.clients.consumer.internals.PartitionAssignor.Subscription
import org.apache.kafka.common.serialization.{ByteArrayDeserializer, StringDeserializer}
import io.circe._
import io.circe.generic.auto._
import cats.data.Xor.{Left, Right}
import model.Employee
import scala.concurrent.Future
object StreamConsumer extends App{
implicit val actorSystem = ActorSystem("consumer-actors", ConfigFactory.load())
implicit val materializer = ActorMaterializer(ActorMaterializerSettings(actorSystem))
lazy val consumerSettings = ConsumerSettings(actorSystem, new ByteArrayDeserializer, new StringDeserializer)
.withBootstrapServers("localhost:9092")
.withGroupId("group13")
.withProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest")//"latest")
.withProperty(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "true")
.withProperty(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, "1000")
lazy val subscription = Subscriptions.topics("raw-data-1")
lazy val db = new Processor()
Consumer.plainSource(consumerSettings, subscription)
.mapAsync(4){
db.processMessage
}
.runWith(Sink.ignore)
}
class Processor {
def processMessage(record: ConsumerRecord[Array[Byte], String]): Future[Done] ={
println(s"DB.save: ${record.value()}")
Option(record.value()).foreach{ jsonString =>
val mayBeEmp: Xor[Error, Employee] = jawn.decode[Employee](jsonString)
mayBeEmp match {
case Left(error) => println(error)
case Right(emp) => println(s"employee name: ${emp.name}")
}
}
Future.successful(Done) }
}
示例6: TopicHandler
//设置package包名称以及导入依赖的类
package org.hpi.esb.datavalidator.kafka
import akka.actor.ActorSystem
import akka.kafka.scaladsl.Consumer
import akka.kafka.{ConsumerSettings, Subscriptions}
import akka.stream.scaladsl.Source
import org.apache.kafka.clients.consumer.{ConsumerConfig, ConsumerRecord}
import org.apache.kafka.common.TopicPartition
import org.apache.kafka.common.serialization.StringDeserializer
import org.hpi.esb.util.OffsetManagement
case class TopicHandler(topicName: String, numberOfMessages: Long, topicSource: Source[ConsumerRecord[String, String], Consumer.Control])
object TopicHandler {
def create(topicName: String, system: ActorSystem): TopicHandler = {
val uuid = java.util.UUID.randomUUID.toString
val consumerSettings: ConsumerSettings[String, String] = ConsumerSettings(system, new StringDeserializer, new StringDeserializer)
.withBootstrapServers("192.168.30.208:9092,192.168.30.207:9092,192.168.30.141:9092")
.withGroupId(uuid)
.withProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest")
.withProperty(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, Int.MaxValue.toString)
.withProperty(ConsumerConfig.FETCH_MAX_BYTES_CONFIG, Int.MaxValue.toString)
.withProperty(ConsumerConfig.FETCH_MIN_BYTES_CONFIG, "20485000")
val partition = 0
val topicSource = createSource(consumerSettings, topicName, partition)
val numberOfMessages = OffsetManagement.getNumberOfMessages(topicName, partition)
new TopicHandler(topicName, numberOfMessages, topicSource)
}
def createSource(consumerSettings: ConsumerSettings[String, String], topicName: String, partition: Int) = {
val subscription = Subscriptions.assignmentWithOffset(
new TopicPartition(topicName, partition) -> 0L
)
Consumer.plainSource(consumerSettings, subscription)
}
}
示例7: Main
//设置package包名称以及导入依赖的类
import akka.actor.ActorSystem
import akka.kafka.{ConsumerSettings, Subscriptions}
import akka.kafka.scaladsl.Consumer
import akka.stream.ActorMaterializer
import org.apache.kafka.clients.consumer.ConsumerConfig
import org.apache.kafka.common.serialization.{ByteArrayDeserializer, StringDeserializer}
import scala.concurrent.Future
object Main {
def main(args: Array[String]): Unit = {
implicit val system = ActorSystem.apply("akka-stream-kafka")
implicit val materializer = ActorMaterializer()
val consumerSettings = ConsumerSettings(system, new ByteArrayDeserializer, new StringDeserializer)
.withBootstrapServers("localhost:9092;localhost:9093")
.withGroupId("group1")
.withProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest")
Consumer.committableSource(consumerSettings, Subscriptions.topics("topic1"))
.mapAsync(1)(msg => {
msg.committableOffset.commitScaladsl
Future.successful(msg)
})
.runForeach(msg => println(s"partition: ${msg.record.partition}; value: ${msg.record.value}"))
}
}
示例8: ReactiveKafkaSingleConsumerMultipleProducerScala
//设置package包名称以及导入依赖的类
package org.rgcase.reactivekafka
import akka.actor.ActorSystem
import akka.kafka.ConsumerMessage.{ CommittableMessage, CommittableOffsetBatch }
import akka.kafka.ProducerMessage.Message
import akka.kafka.scaladsl.{ Consumer, Producer }
import akka.kafka.{ ConsumerSettings, ProducerSettings, Subscriptions }
import akka.stream.ActorMaterializer
import akka.stream.scaladsl.{ Flow, Sink }
import org.apache.kafka.clients.consumer.ConsumerConfig
import org.apache.kafka.clients.producer.ProducerRecord
import org.apache.kafka.common.serialization.{ ByteArrayDeserializer, ByteArraySerializer, StringDeserializer, StringSerializer }
class ReactiveKafkaSingleConsumerMultipleProducerScala extends App {
implicit val system = ActorSystem("reactivekafkascala")
implicit val mat = ActorMaterializer()
val consumerSettings = ConsumerSettings(system, new ByteArrayDeserializer, new StringDeserializer)
.withBootstrapServers("localhost:9092")
.withGroupId("group1")
.withProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest")
val producerSettings = ProducerSettings(system, new ByteArraySerializer, new StringSerializer)
.withBootstrapServers("localhost:9093")
val kafkaSource =
Consumer.committableSource(consumerSettings, Subscriptions.topics("sourcetopic"))
def toProducerMessage(topic: String) = (msg: CommittableMessage[Array[Byte], String]) ?
Message[Array[Byte], String, CommittableMessage[Array[Byte], String]](new ProducerRecord(topic, msg.record.value), msg)
val producerFlow1 =
Flow.fromFunction(toProducerMessage("targettopic1")).via(Producer.flow(producerSettings)).map(_.message.passThrough)
val producerFlow2 =
Flow.fromFunction(toProducerMessage("targettopic2")).via(Producer.flow(producerSettings)).map(_.message.passThrough)
val producerFlow3 =
Flow.fromFunction(toProducerMessage("targettopic3")).via(Producer.flow(producerSettings)).map(_.message.passThrough)
kafkaSource
.via(producerFlow1)
.via(producerFlow2)
.via(producerFlow3)
.batch(max = 20, first ? CommittableOffsetBatch.empty.updated(first.committableOffset)) { (batch, elem) ?
batch.updated(elem.committableOffset)
}.mapAsync(3)(_.commitScaladsl())
.runWith(Sink.ignore)
}
开发者ID:rgcase,项目名称:testplayground,代码行数:52,代码来源:ReactiveKafkaSingleConsumerMultipleProducerScala.scala
示例9: BatchCommittableSourceConsumerMain
//设置package包名称以及导入依赖的类
package com.example.consumer
import akka.Done
import akka.actor.ActorSystem
import akka.kafka.ConsumerMessage.CommittableOffsetBatch
import akka.kafka.scaladsl.Consumer
import akka.kafka.{ConsumerSettings, Subscriptions}
import akka.stream.ActorMaterializer
import akka.stream.scaladsl.Sink
import org.apache.kafka.clients.consumer.ConsumerConfig
import org.apache.kafka.common.serialization.{ByteArrayDeserializer, StringDeserializer}
import scala.concurrent.ExecutionContext.Implicits.global
import scala.concurrent.Future
object BatchCommittableSourceConsumerMain extends App {
implicit val system = ActorSystem("BatchCommittableSourceConsumerMain")
implicit val materializer = ActorMaterializer()
//TODO: move to configuration application.conf
val consumerSettings = ConsumerSettings(system, new ByteArrayDeserializer, new StringDeserializer)
.withBootstrapServers("localhost:9092")
.withGroupId("BatchCommittableSourceConsumer")
.withProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest")
val done =
Consumer.committableSource(consumerSettings, Subscriptions.topics("topic1"))
.mapAsync(1) { msg =>
println(s"BatchCommittableConsumer consume: $msg")
Future.successful(Done).map(_ => msg.committableOffset)
}
.batch(max = 20, first => CommittableOffsetBatch.empty.updated(first)) { (batch, elem) =>
batch.updated(elem)
}
.mapAsync(3)(_.commitScaladsl())
.runWith(Sink.ignore)
}
开发者ID:makersu,项目名称:reactive-kafka-scala-example,代码行数:39,代码来源:BatchCommittableSourceConsumerMain.scala
示例10: PlainSourceConsumerMain
//设置package包名称以及导入依赖的类
package com.example.consumer
import java.util.concurrent.atomic.AtomicLong
import akka.Done
import akka.actor.ActorSystem
import akka.kafka.scaladsl.Consumer
import akka.kafka.{ConsumerSettings, Subscriptions}
import akka.stream.ActorMaterializer
import akka.stream.scaladsl.Sink
import org.apache.kafka.clients.consumer.{ConsumerConfig, ConsumerRecord}
import org.apache.kafka.common.TopicPartition
import org.apache.kafka.common.serialization.{ByteArrayDeserializer, StringDeserializer}
import scala.concurrent.ExecutionContext.Implicits.global
import scala.concurrent.Future
object PlainSourceConsumerMain extends App {
implicit val system = ActorSystem("PlainSourceConsumerMain")
implicit val materializer = ActorMaterializer()
//TODO: move to configuration application.conf
val consumerSettings = ConsumerSettings(system, new ByteArrayDeserializer, new StringDeserializer)
.withBootstrapServers("localhost:9092")
.withGroupId("PlainSourceConsumer")
.withProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest")
val db = new DB
db.loadOffset().foreach { fromOffset =>
val partition = 0
val subscription = Subscriptions.assignmentWithOffset(
new TopicPartition("topic1", partition) -> fromOffset
)
val done =
Consumer.plainSource(consumerSettings, subscription)
.mapAsync(1)(db.save)
.runWith(Sink.ignore)
}
}
//External Offset Storage
class DB {
private val offset = new AtomicLong(2)
def save(record: ConsumerRecord[Array[Byte], String]): Future[Done] = {
println(s"DB.save: ${record.value}")
offset.set(record.offset)
Future.successful(Done)
}
def loadOffset(): Future[Long] =
Future.successful(offset.get)
def update(data: String): Future[Done] = {
println(s"DB.update: $data")
Future.successful(Done)
}
}
示例11: CommitConsumerToFlowProducerMain
//设置package包名称以及导入依赖的类
package com.example.producer
import akka.actor.ActorSystem
import akka.kafka.scaladsl.{Consumer, Producer}
import akka.kafka.{ConsumerSettings, ProducerMessage, ProducerSettings, Subscriptions}
import akka.stream.ActorMaterializer
import akka.stream.scaladsl.Sink
import org.apache.kafka.clients.consumer.ConsumerConfig
import org.apache.kafka.clients.producer.ProducerRecord
import org.apache.kafka.common.serialization.{ByteArrayDeserializer, ByteArraySerializer, StringDeserializer, StringSerializer}
object CommitConsumerToFlowProducerMain extends App {
implicit val system = ActorSystem("CommitConsumerToFlowProducerMain")
implicit val materializer = ActorMaterializer()
val consumerSettings =
ConsumerSettings(system, new ByteArrayDeserializer, new StringDeserializer)
.withBootstrapServers("localhost:9092")
.withGroupId("CommitConsumerToFlowProducer")
.withProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest")
val producerSettings = ProducerSettings(system, new ByteArraySerializer, new StringSerializer)
.withBootstrapServers("localhost:9092")
val done =
Consumer.committableSource(consumerSettings, Subscriptions.topics("topic1"))
.map { msg =>
println(s"topic1 -> topic2: $msg")
ProducerMessage.Message(new ProducerRecord[Array[Byte], String](
"topic2",
msg.record.value
), msg.committableOffset)
}
.via(Producer.flow(producerSettings))
.mapAsync(producerSettings.parallelism) { result =>
result.message.passThrough.commitScaladsl()
}
.runWith(Sink.ignore)
}
示例12: ConsumerToCommitableSinkProducerMain
//设置package包名称以及导入依赖的类
package com.example.producer
import akka.actor.ActorSystem
import akka.kafka.ConsumerMessage.CommittableOffsetBatch
import akka.kafka.scaladsl.{Consumer, Producer}
import akka.kafka.{ConsumerSettings, ProducerMessage, ProducerSettings, Subscriptions}
import akka.stream.ActorMaterializer
import akka.stream.scaladsl.Sink
import org.apache.kafka.clients.consumer.ConsumerConfig
import org.apache.kafka.clients.producer.ProducerRecord
import org.apache.kafka.common.serialization.{ByteArrayDeserializer, ByteArraySerializer, StringDeserializer, StringSerializer}
object ConsumerToCommitableSinkProducerMain extends App {
implicit val system = ActorSystem("Consumer2ProducerMain")
implicit val materializer = ActorMaterializer()
//TODO: move to configuration application.conf
val consumerSettings =
ConsumerSettings(system, new ByteArrayDeserializer, new StringDeserializer)
.withBootstrapServers("localhost:9092")
.withGroupId("Consumer2Producer")
.withProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest")
//TODO: move to configuration application.conf
val producerSettings =
ProducerSettings(system, new ByteArraySerializer, new StringSerializer)
.withBootstrapServers("localhost:9092")
Consumer.committableSource(consumerSettings, Subscriptions.topics("topic1"))
.map { msg =>
println(s"topic1 -> topic2: $msg")
ProducerMessage.Message(new ProducerRecord[Array[Byte], String](
"topic2",
msg.record.value
), msg.committableOffset)
}
.runWith(Producer.commitableSink(producerSettings))
}
开发者ID:makersu,项目名称:reactive-kafka-scala-example,代码行数:42,代码来源:ConsumerToCommitableSinkProducerMain.scala
示例13: KafkaSettings
//设置package包名称以及导入依赖的类
package com.ovoenergy.orchestration.kafka
import java.nio.file.Paths
import akka.actor.ActorSystem
import akka.kafka.ConsumerSettings
import com.ovoenergy.comms.akka.streams.Factory
import com.ovoenergy.comms.akka.streams.Factory.{KafkaConfig, SSLConfig}
import com.ovoenergy.comms.serialisation.Serialisation.avroDeserializer
import com.ovoenergy.kafka.serialization.avro.SchemaRegistryClientSettings
import com.sksamuel.avro4s.{FromRecord, SchemaFor}
import com.typesafe.config.Config
import org.apache.kafka.common.serialization.StringDeserializer
import scala.reflect.ClassTag
class KafkaSettings(config: Config) {
private val kafkaGroupId = config.getString("kafka.group.id")
val schemaRegistryClientSettings = {
val schemaRegistryEndpoint = config.getString("kafka.aiven.schema_registry.url")
val schemaRegistryUsername = config.getString("kafka.aiven.schema_registry.username")
val schemaRegistryPassword = config.getString("kafka.aiven.schema_registry.password")
SchemaRegistryClientSettings(schemaRegistryEndpoint, schemaRegistryUsername, schemaRegistryPassword)
}
def getAivenKafkaConfig(topicKey: String) = {
val topic = config.getString(topicKey)
KafkaConfig(kafkaGroupId, config.getString("kafka.aiven.hosts"), topic, sslConfig)
}
def getLegacyKafkaConfig(topicKey: String) = {
val topic = config.getString(topicKey)
KafkaConfig(kafkaGroupId, config.getString("kafka.hosts"), topic, None)
}
val sslConfig = {
if (config.getBoolean("kafka.aiven.ssl.enabled")) {
Some(
Factory.SSLConfig(
keystoreLocation = Paths.get(config.getString("kafka.aiven.ssl.keystore.location")),
keystoreType = Factory.StoreType.PKCS12,
keystorePassword = config.getString("kafka.aiven.ssl.keystore.password"),
keyPassword = config.getString("kafka.aiven.ssl.key.password"),
truststoreLocation = Paths.get(config.getString("kafka.aiven.ssl.truststore.location")),
truststoreType = Factory.StoreType.JKS,
truststorePassword = config.getString("kafka.aiven.ssl.truststore.password")
))
} else None
}
def legacyConsumerSettings[T: SchemaFor: FromRecord: ClassTag](config: KafkaConfig)(
implicit actorSystem: ActorSystem) = {
ConsumerSettings(actorSystem, new StringDeserializer, avroDeserializer[T])
.withBootstrapServers(config.hosts)
.withGroupId(config.groupId)
}
}
示例14: produceRecord
//设置package包名称以及导入依赖的类
package de.choffmeister.microserviceutils.kafka.testkit
import java.util.UUID
import akka.Done
import akka.actor.ActorSystem
import akka.kafka.scaladsl.{Consumer, Producer}
import akka.kafka.{ConsumerSettings, ProducerSettings, Subscriptions}
import akka.stream.Materializer
import akka.stream.scaladsl.{Keep, Sink, Source}
import org.apache.kafka.clients.consumer.{ConsumerConfig, ConsumerRecord}
import org.apache.kafka.clients.producer.ProducerRecord
import org.apache.kafka.common.serialization.{Deserializer, Serializer}
import scala.concurrent.Future
trait KafkaTest {
def produceRecord[K, V](topic: String, keySerializer: Serializer[K], valueSerializer: Serializer[V], key: K, value: V)(implicit system: ActorSystem, mat: Materializer): Future[Done] = {
val producerSettings = ProducerSettings(system, keySerializer, valueSerializer)
.withBootstrapServers(system.settings.config.getString("kafka.bootstrap-servers"))
Source.single(new ProducerRecord("mail.command.send", key, value))
.toMat(Producer.plainSink(producerSettings))(Keep.right)
.run()
}
def consumeRecordPF[K, V, Out](topic: String, keyDeserializer: Deserializer[K], valueDeserializer: Deserializer[V])(pf: PartialFunction[ConsumerRecord[K, V], Out])(implicit system: ActorSystem, mat: Materializer): Future[Out] = {
val consumerSettings = ConsumerSettings(system, keyDeserializer, valueDeserializer)
.withBootstrapServers(system.settings.config.getString("kafka.bootstrap-servers"))
.withGroupId(UUID.randomUUID.toString)
.withProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest")
Consumer.plainSource(consumerSettings, Subscriptions.topics(topic))
.filter(pf.isDefinedAt)
.map(pf)
.toMat(Sink.head)(Keep.right)
.run()
}
}
示例15: Config
//设置package包名称以及导入依赖的类
package com.github.kliewkliew.cornucopia.kafka
import akka.actor.ActorSystem
import akka.kafka.scaladsl.{Producer, Consumer => ConsumerDSL}
import akka.kafka.{ConsumerSettings, ProducerSettings, Subscriptions}
import akka.stream.{ActorMaterializer, ActorMaterializerSettings, Supervision}
import com.typesafe.config.ConfigFactory
import org.apache.kafka.common.serialization.{StringDeserializer, StringSerializer}
import org.slf4j.LoggerFactory
import scala.concurrent.duration._
object Config {
object Cornucopia {
private val config = ConfigFactory.load().getConfig("cornucopia")
val minReshardWait = config.getInt("reshard.interval").seconds
val gracePeriod = config.getInt("grace.period") * 1000
val refreshTimeout = config.getInt("refresh.timeout") * 1000
val batchPeriod = config.getInt("batch.period").seconds
}
object Consumer {
private val kafkaConfig = ConfigFactory.load().getConfig("kafka")
private val kafkaServers = kafkaConfig.getString("bootstrap.servers")
private val kafkaConsumerConfig = kafkaConfig.getConfig("consumer")
private val topic = kafkaConsumerConfig.getString("topic")
private val groupId = kafkaConsumerConfig.getString("group.id")
implicit val actorSystem = ActorSystem()
// Log failures and resume processing
private val decider: Supervision.Decider = { e =>
LoggerFactory.getLogger(this.getClass).error("Failed to process event", e)
Supervision.Resume
}
private val materializerSettings = ActorMaterializerSettings(actorSystem).withSupervisionStrategy(decider)
private val sourceSettings = ConsumerSettings(actorSystem, new StringDeserializer, new StringDeserializer)
.withBootstrapServers(kafkaServers)
.withGroupId(groupId)
private val subscription = Subscriptions.topics(topic)
private val sinkSettings = ProducerSettings(actorSystem, new StringSerializer, new StringSerializer)
.withBootstrapServers(kafkaServers)
implicit val materializer = ActorMaterializer(materializerSettings)(actorSystem)
val cornucopiaSource = ConsumerDSL.plainSource(sourceSettings, subscription)
val cornucopiaSink = Producer.plainSink(sinkSettings)
}
}